From 8fee1bc41eb541fdcf36f174792cd837b8b65412 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Sun, 27 Sep 2026 23:36:31 -0700 Subject: [PATCH 01/30] fix(usage): store disjoint input, cache, and output token counters Collectors now report input, cache read, cache write, and output as disjoint counters with reasoning as a breakdown of output. Derive a missing total from those four counters instead of also adding reasoning, and return output and cache counters from the token_usage summary projection so charts can split cached input from uncached input. Rewrite stored rows for both SQLite and Postgres: remove OpenAI-style cached input from input_tokens and recompute total_tokens, which older collectors set to input_tokens alone and so dropped Anthropic cache reads and writes. Co-Authored-By: Claude Opus 5.5 --- .../down.sql | 2 + .../up.sql | 18 ++++++++ .../down.sql | 2 + .../up.sql | 18 ++++++++ crates/abyss-backend/src/usage/mod.rs | 34 ++++++++++---- crates/abyss-backend/src/usage/persistence.rs | 44 +++++++++++-------- 6 files changed, 90 insertions(+), 28 deletions(-) create mode 100644 crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql create mode 100644 crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql create mode 100644 crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql create mode 100644 crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql diff --git a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql new file mode 100644 index 0000000..0969841 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql @@ -0,0 +1,2 @@ +-- The original input/total split cannot be recovered after the rewrite. +SELECT 1; diff --git a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql new file mode 100644 index 0000000..acadb91 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql @@ -0,0 +1,18 @@ +-- Older collectors counted OpenAI-style cached input inside input_tokens and +-- set request totals to input_tokens alone, dropping Anthropic cache reads and +-- writes. Rewrite stored rows to disjoint input, cache read, cache write, and +-- output counters with total_tokens as their sum. +-- +-- Anthropic Messages already reports cache counters outside input_tokens. +-- Provider names cannot identify the protocol because gateway traffic records +-- the gateway host, so Anthropic harnesses are excluded by agent name as well, +-- and rows whose input is smaller than their cached input are left untouched. +UPDATE llm_usage_events +SET input_tokens = input_tokens - cache_read_tokens - cache_write_tokens +WHERE llm_provider <> 'anthropic' + AND agent_name NOT IN ('claude-code', 'claude-desktop') + AND cache_read_tokens + cache_write_tokens > 0 + AND input_tokens >= cache_read_tokens + cache_write_tokens; + +UPDATE llm_usage_events +SET total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens; diff --git a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql new file mode 100644 index 0000000..0969841 --- /dev/null +++ b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql @@ -0,0 +1,2 @@ +-- The original input/total split cannot be recovered after the rewrite. +SELECT 1; diff --git a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql new file mode 100644 index 0000000..acadb91 --- /dev/null +++ b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql @@ -0,0 +1,18 @@ +-- Older collectors counted OpenAI-style cached input inside input_tokens and +-- set request totals to input_tokens alone, dropping Anthropic cache reads and +-- writes. Rewrite stored rows to disjoint input, cache read, cache write, and +-- output counters with total_tokens as their sum. +-- +-- Anthropic Messages already reports cache counters outside input_tokens. +-- Provider names cannot identify the protocol because gateway traffic records +-- the gateway host, so Anthropic harnesses are excluded by agent name as well, +-- and rows whose input is smaller than their cached input are left untouched. +UPDATE llm_usage_events +SET input_tokens = input_tokens - cache_read_tokens - cache_write_tokens +WHERE llm_provider <> 'anthropic' + AND agent_name NOT IN ('claude-code', 'claude-desktop') + AND cache_read_tokens + cache_write_tokens > 0 + AND input_tokens >= cache_read_tokens + cache_write_tokens; + +UPDATE llm_usage_events +SET total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens; diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 0d055c2..bb9c223 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -118,18 +118,22 @@ impl UsageEventType { clippy::struct_field_names, reason = "Public JSON token usage fields intentionally include the token suffix." )] +/// Token counters reported by the event producer. +/// +/// Input, cache read, cache write, and output counters are disjoint; +/// reasoning is a breakdown of output. pub struct TokenUsagePayload { - /// Input tokens reported by the provider. + /// Input tokens neither read from nor written into a provider cache. pub input_tokens: Option, - /// Output tokens reported by the provider. + /// Output tokens, including reasoning. pub output_tokens: Option, - /// Tokens served from a provider cache. + /// Input tokens served from a provider cache. pub cache_read_tokens: Option, - /// Tokens written to a provider cache. + /// Input tokens written to a provider cache. pub cache_write_tokens: Option, - /// Provider-reported reasoning tokens. + /// Part of output tokens spent on reasoning. pub reasoning_tokens: Option, - /// Provider total, or a derived sum when omitted. + /// Sum of the four disjoint counters, derived when omitted. pub total_tokens: Option, } @@ -183,7 +187,7 @@ pub struct SummaryQuery { pub enum SummaryFields { /// Return dimensions, event counts, and every token counter. Full, - /// Return only day, input-token, and total-token fields. + /// Return only the day and disjoint token counters used by charts. TokenUsage, } @@ -298,9 +302,15 @@ pub struct TokenUsageSummaryRow { /// UTC calendar day when day grouping was requested. #[serde(skip_serializing_if = "Option::is_none")] pub day: Option, - /// Summed input tokens. + /// Summed input tokens that were neither read from nor written to a cache. pub input_tokens: i64, - /// Summed provider or derived total tokens. + /// Summed output tokens, including reasoning. + pub output_tokens: i64, + /// Summed input tokens read from a provider cache. + pub cache_read_tokens: i64, + /// Summed input tokens written into a provider cache. + pub cache_write_tokens: i64, + /// Summed total of the four disjoint counters. pub total_tokens: i64, } @@ -309,6 +319,9 @@ impl From for TokenUsageSummaryRow { Self { day: row.day, input_tokens: row.input_tokens, + output_tokens: row.output_tokens, + cache_read_tokens: row.cache_read_tokens, + cache_write_tokens: row.cache_write_tokens, total_tokens: row.total_tokens, } } @@ -668,6 +681,9 @@ mod tests { { "day": "2026-07-01", "input_tokens": 123_i64, + "output_tokens": 456_i64, + "cache_read_tokens": 11_i64, + "cache_write_tokens": 12_i64, "total_tokens": 615_i64 } ], diff --git a/crates/abyss-backend/src/usage/persistence.rs b/crates/abyss-backend/src/usage/persistence.rs index 1112eae..9497eb0 100644 --- a/crates/abyss-backend/src/usage/persistence.rs +++ b/crates/abyss-backend/src/usage/persistence.rs @@ -67,11 +67,7 @@ pub fn validate_event(event: &IngestUsageEvent) -> Result<(), AppError> { if tokens.total == 0 { return Ok(()); } - let visible_tokens = tokens - .input - .saturating_add(tokens.output) - .saturating_add(tokens.reasoning); - if tokens.total < visible_tokens { + if tokens.total < tokens.component_total() { tracing::debug!( event_id = %event.event_id, "provider total_tokens is lower than visible token components" @@ -96,31 +92,41 @@ pub struct NormalizedTokens { pub total: i64, } +impl NormalizedTokens { + /// Sum of the disjoint input, cache read, cache write, and output counters. + /// + /// Reasoning is a breakdown of output and is never added again. + pub const fn component_total(&self) -> i64 { + self.input + .saturating_add(self.cache_read) + .saturating_add(self.cache_write) + .saturating_add(self.output) + } +} + pub fn normalized_tokens(event: &IngestUsageEvent) -> Result { let input = normalize_token(event.token_usage.input_tokens, "input_tokens")?; let output = normalize_token(event.token_usage.output_tokens, "output_tokens")?; let cache_read = normalize_token(event.token_usage.cache_read_tokens, "cache_read_tokens")?; let cache_write = normalize_token(event.token_usage.cache_write_tokens, "cache_write_tokens")?; let reasoning = normalize_token(event.token_usage.reasoning_tokens, "reasoning_tokens")?; - // Preserve an explicit provider total because provider accounting may not - // equal the visible component sum; derive only when the field is absent. - let total = match event.token_usage.total_tokens { - Some(value) => normalize_token(Some(value), "total_tokens")?, - None => input - .saturating_add(output) - .saturating_add(cache_read) - .saturating_add(cache_write) - .saturating_add(reasoning), - }; - - Ok(NormalizedTokens { + let explicit_total = event + .token_usage + .total_tokens + .map(|value| normalize_token(Some(value), "total_tokens")) + .transpose()?; + let mut tokens = NormalizedTokens { input, output, cache_read, cache_write, reasoning, - total, - }) + total: 0, + }; + // Preserve an explicit producer total; derive it from the disjoint + // counters only when the field is absent. + tokens.total = explicit_total.unwrap_or_else(|| tokens.component_total()); + Ok(tokens) } fn normalize_token(value: Option, field: &str) -> Result { From c8fba99fdbe58d8791beef9d7b2de17e3ebfc5f2 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Mon, 28 Sep 2026 02:33:45 -0700 Subject: [PATCH 02/30] feat: query single events, session outlines, and merged trajectories Viewing a large session loaded every accumulated request snapshot at once (66 MB for a 1,590-event session). Add three read paths so clients can load only what they show: - GET /v1/agent-usage/events/{event_pk} returns one event verbatim. - GET /v1/agent-usage/sessions/{session_pk}/outline returns the timeline without text or metadata bodies, plus text length, a short preview, and attachment counts (about 1 MB for the same session). - GET /v1/agent-usage/sessions/{session_pk}/trajectory merges the timeline so each request and response contributes only its new content: new request paragraphs, response text, and tool calls paired with their first-seen results. Side requests such as classifiers form their own streams. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/api.rs | 43 +- crates/abyss-backend/src/storage/mod.rs | 17 +- crates/abyss-backend/src/storage/postgres.rs | 27 +- .../src/storage/postgres/repository.rs | 188 +++++++- .../abyss-backend/src/storage/sqlite/mod.rs | 27 +- .../src/storage/sqlite/repository.rs | 268 +++++++++-- crates/abyss-backend/src/usage/event_order.rs | 22 + crates/abyss-backend/src/usage/mod.rs | 83 +++- crates/abyss-backend/src/usage/trajectory.rs | 422 ++++++++++++++++++ scripts/tests/blackbox_api.py | 77 ++++ 10 files changed, 1120 insertions(+), 54 deletions(-) create mode 100644 crates/abyss-backend/src/usage/trajectory.rs diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index 35a6629..59c3213 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -23,7 +23,9 @@ use crate::{ storage::StorageBackend, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryFields, SummaryQuery, TokenUsageSummaryResponse, + SessionOutlineResponse, SessionTimelineResponse, SummaryFields, SummaryQuery, + TokenUsageSummaryResponse, UsageEventResponse, + trajectory::{self, SessionTrajectoryResponse}, }, }; @@ -61,6 +63,7 @@ pub fn router(state: AppState) -> Router { .get(raw_events) .layer(DefaultBodyLimit::max(MAX_INGEST_REQUEST_BODY_BYTES)), ) + .route("/v1/agent-usage/events/{event_pk}", get(usage_event)) .route( "/v1/agent-usage/attachments/{attachment_id}", get(image_attachment), @@ -71,6 +74,14 @@ pub fn router(state: AppState) -> Router { "/v1/agent-usage/sessions/{session_pk}", get(session_timeline), ) + .route( + "/v1/agent-usage/sessions/{session_pk}/outline", + get(session_outline), + ) + .route( + "/v1/agent-usage/sessions/{session_pk}/trajectory", + get(session_trajectory), + ) .layer(TraceLayer::new_for_http()) .with_state(state) } @@ -152,6 +163,36 @@ async fn session_timeline( Ok(Json(response)) } +async fn session_outline( + State(state): State, + headers: HeaderMap, + Path(session_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + Ok(Json( + state.storage.session_outline(user_id, session_pk).await?, + )) +} + +async fn session_trajectory( + State(state): State, + headers: HeaderMap, + Path(session_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + let timeline = state.storage.session_timeline(user_id, session_pk).await?; + Ok(Json(trajectory::build(timeline))) +} + +async fn usage_event( + State(state): State, + headers: HeaderMap, + Path(event_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + Ok(Json(state.storage.usage_event(user_id, event_pk).await?)) +} + async fn image_attachment( State(state): State, headers: HeaderMap, diff --git a/crates/abyss-backend/src/storage/mod.rs b/crates/abyss-backend/src/storage/mod.rs index 7cb6aa6..ebf0d8e 100644 --- a/crates/abyss-backend/src/storage/mod.rs +++ b/crates/abyss-backend/src/storage/mod.rs @@ -20,7 +20,8 @@ use crate::{ search::{SessionSearchQuery, SessionSearchResponse}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, attachments::StoredImageAttachment, }, }; @@ -48,6 +49,13 @@ pub trait StorageBackend: Send + Sync { default_page_size: i64, ) -> BoxedFuture<'_, Result>; + /// Loads one owner-scoped request or response event with its full content. + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result>; + /// Aggregates owner-scoped event and token usage. fn usage_summary( &self, @@ -63,6 +71,13 @@ pub trait StorageBackend: Send + Sync { session_pk: Uuid, ) -> BoxedFuture<'_, Result>; + /// Loads one owner-scoped session timeline without event content bodies. + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result>; + /// Loads authorized image bytes by attachment identifier. fn image_attachment( &self, diff --git a/crates/abyss-backend/src/storage/postgres.rs b/crates/abyss-backend/src/storage/postgres.rs index e9716cd..f267bd8 100644 --- a/crates/abyss-backend/src/storage/postgres.rs +++ b/crates/abyss-backend/src/storage/postgres.rs @@ -17,7 +17,8 @@ use crate::{ storage::{BoxedFuture, StorageBackend}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, attachments::StoredImageAttachment, }, }; @@ -113,6 +114,17 @@ impl StorageBackend for PostgresEsBackend { }) } + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| repository::usage_event(connection, user_id, event_pk)) + .await + }) + } + fn usage_summary( &self, user_id: Uuid, @@ -140,6 +152,19 @@ impl StorageBackend for PostgresEsBackend { }) } + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| { + repository::session_outline(connection, user_id, session_pk) + }) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index df586b8..8c1c052 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -11,7 +11,7 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ Connection, ExpressionMethods, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, - dsl::{max, sql}, + dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, sql_types::{ @@ -35,9 +35,10 @@ use crate::{ }, error::AppError, usage::{ - IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, RawEventsQuery, - RawEventsResponse, SessionInfo, SessionTimelineResponse, SummaryQuery, SummaryResponse, - SummaryRow, TurnTimeline, UsageEventResponse, + IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, OUTLINE_PREVIEW_CHARS, + RawEventsQuery, RawEventsResponse, SessionInfo, SessionOutlineResponse, + SessionTimelineResponse, SummaryQuery, SummaryResponse, SummaryRow, TurnTimeline, + UsageEventOutline, UsageEventResponse, attachments::{ ImageMediaType, StoredImageAttachment, UsageEventAttachmentResponse, ValidatedImageAttachment, @@ -347,20 +348,7 @@ pub fn session_timeline( user_id: Uuid, session_pk: Uuid, ) -> Result { - // Timeline views include prompt/response text, so they are always limited - // to the owner. - let session = agent_sessions::table - .filter(agent_sessions::id.eq(session_pk)) - .filter(agent_sessions::user_id.eq(user_id)) - .first::(connection) - .optional()? - .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))?; - touch_session_metadata(&session); - - let turns = agent_turns::table - .filter(agent_turns::session_pk.eq(session.id)) - .order(agent_turns::turn_index.asc()) - .load::(connection)?; + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; let mut events = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(session.id)) @@ -386,7 +374,140 @@ pub fn session_timeline( attachments_by_event.remove(&event_pk).unwrap_or_default(), )); } + Ok(assemble_timeline(session, turns, events_by_turn)) +} + +type OutlineRow = ( + Uuid, + String, + Uuid, + i32, + String, + String, + String, + i64, + i64, + i64, + i64, + i64, + i64, + bool, + DateTime, + i64, + Option, + Option, + Option, +); + +/// Loads one owner-scoped session timeline without event content bodies. +pub fn session_outline( + connection: &mut PgConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result { + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; + // Only fixed-size columns and SQL-computed summaries are selected, so the + // outline stays small no matter how large event text or metadata grows. + let rows = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(session.id)) + .select(( + llm_usage_events::id, + llm_usage_events::event_id, + llm_usage_events::turn_pk, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + sql::("coalesce(metadata -> 'token_usage_estimated' = 'true'::jsonb, false)"), + llm_usage_events::observed_at, + sql::("coalesce(char_length(text), 0)::bigint"), + sql::>(&format!( + "CASE WHEN event_type = 'request' THEN right(text, {OUTLINE_PREVIEW_CHARS}) \ + ELSE left(text, {OUTLINE_PREVIEW_CHARS}) END" + )), + sql::>( + "CASE WHEN jsonb_typeof(metadata -> 'response_id') = 'string' \ + THEN metadata ->> 'response_id' END", + ), + sql::>( + "CASE WHEN jsonb_typeof(metadata -> 'previous_response_id') = 'string' \ + THEN metadata ->> 'previous_response_id' END", + ), + )) + .load::(connection)?; + let event_ids = rows.iter().map(|row| row.0).collect::>(); + let attachment_counts = llm_usage_event_attachments::table + .filter(llm_usage_event_attachments::event_pk.eq_any(event_ids)) + .group_by(llm_usage_event_attachments::event_pk) + .select((llm_usage_event_attachments::event_pk, count_star())) + .load::<(Uuid, i64)>(connection)? + .into_iter() + .collect::>(); + let mut events = rows + .into_iter() + .map(|row| UsageEventOutline { + id: row.0, + event_id: row.1, + turn_pk: row.2, + turn_index: row.3, + llm_provider: row.4, + llm_model: row.5, + event_type: row.6, + input_tokens: row.7, + output_tokens: row.8, + cache_read_tokens: row.9, + cache_write_tokens: row.10, + reasoning_tokens: row.11, + total_tokens: row.12, + token_usage_estimated: row.13, + observed_at: row.14, + text_chars: row.15, + preview: row.16, + attachment_count: attachment_counts.get(&row.0).copied().unwrap_or(0), + ordering_metadata: UsageEventOutline::ordering_metadata(row.17, row.18), + }) + .collect::>(); + UsageEventTimelineOrder::sort(&mut events); + let mut events_by_turn: HashMap> = HashMap::new(); + for event in events { + events_by_turn.entry(event.turn_pk).or_default().push(event); + } + Ok(assemble_timeline(session, turns, events_by_turn)) +} + +fn load_session_turns( + connection: &mut PgConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result<(AgentSession, Vec), AppError> { + // Timeline views include prompt/response text, so they are always limited + // to the owner. + let session = agent_sessions::table + .filter(agent_sessions::id.eq(session_pk)) + .filter(agent_sessions::user_id.eq(user_id)) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))?; + touch_session_metadata(&session); + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session.id)) + .order(agent_turns::turn_index.asc()) + .load::(connection)?; + Ok((session, turns)) +} + +fn assemble_timeline( + session: AgentSession, + turns: Vec, + mut events_by_turn: HashMap>, +) -> SessionTimelineResponse { let turn_timelines = turns .into_iter() .map(|turn| { @@ -401,7 +522,7 @@ pub fn session_timeline( }) .collect(); - Ok(SessionTimelineResponse { + SessionTimelineResponse { session: SessionInfo { session_pk: session.id, user_id: session.user_id, @@ -414,7 +535,7 @@ pub fn session_timeline( metadata: session.metadata, }, turns: turn_timelines, - }) + } } /// Returns one newest-first page of raw owner-scoped events. @@ -474,6 +595,33 @@ pub fn raw_events( }) } +/// Loads one owner-scoped event with the same shape used by timelines. +pub fn usage_event( + connection: &mut PgConnection, + user_id: Uuid, + event_pk: Uuid, +) -> Result { + // Event content can contain sensitive conversation context. Keep this + // endpoint scoped to the deployment owner. + let event = llm_usage_events::table + .filter(llm_usage_events::id.eq(event_pk)) + .filter(llm_usage_events::user_id.eq(user_id)) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("usage event not found: {event_pk}")))?; + let events = [event]; + let device_contexts = load_device_contexts_for_events(connection, &events)?; + let mut attachments_by_event = load_attachments_for_events(connection, &events)?; + let [event] = events; + let event_pk = event.id; + let device_context_id = event.device_context_id; + Ok(event_response( + event, + device_contexts.get(&device_context_id), + attachments_by_event.remove(&event_pk).unwrap_or_default(), + )) +} + /// Loads authorized attachment bytes and verifies persisted size metadata. pub fn image_attachment( connection: &mut PgConnection, diff --git a/crates/abyss-backend/src/storage/sqlite/mod.rs b/crates/abyss-backend/src/storage/sqlite/mod.rs index 4214d9c..431b28c 100644 --- a/crates/abyss-backend/src/storage/sqlite/mod.rs +++ b/crates/abyss-backend/src/storage/sqlite/mod.rs @@ -17,7 +17,8 @@ use crate::{ storage::{BoxedFuture, StorageBackend}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, attachments::StoredImageAttachment, }, }; @@ -97,6 +98,17 @@ impl StorageBackend for SqliteFtsBackend { }) } + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| repository::usage_event(connection, user_id, event_pk)) + .await + }) + } + fn usage_summary( &self, user_id: Uuid, @@ -124,6 +136,19 @@ impl StorageBackend for SqliteFtsBackend { }) } + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| { + repository::session_outline(connection, user_id, session_pk) + }) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index 4c24f29..f21bbc8 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -4,8 +4,11 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper, SqliteConnection, dsl::max, - result::OptionalExtension, sql_query, sql_types::Text, + ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper, SqliteConnection, + dsl::{count_star, max, sql}, + result::OptionalExtension, + sql_query, + sql_types::{BigInt, Bool, Nullable, Text}, }; use uuid::Uuid; @@ -13,9 +16,10 @@ use crate::{ error::AppError, search::projection::SearchProjection, usage::{ - IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, RawEventsQuery, - RawEventsResponse, SessionInfo, SessionTimelineResponse, SummaryQuery, SummaryResponse, - SummaryRow, TurnTimeline, UsageEventResponse, + IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, OUTLINE_PREVIEW_CHARS, + RawEventsQuery, RawEventsResponse, SessionInfo, SessionOutlineResponse, + SessionTimelineResponse, SummaryQuery, SummaryResponse, SummaryRow, TurnTimeline, + UsageEventOutline, UsageEventResponse, attachments::{ ImageMediaType, StoredImageAttachment, UsageEventAttachmentResponse, ValidatedImageAttachment, @@ -35,8 +39,8 @@ use crate::{ use super::{ models::{ AttachmentRow, DeviceRecord, DeviceRow, EventRecord, EventRow, NewAttachment, NewDevice, - NewDiagnosticCapture, NewDiagnosticCaptureEvent, NewEvent, NewSession, NewTurn, SessionRow, - TurnRow, parse_uuid, + NewDiagnosticCapture, NewDiagnosticCaptureEvent, NewEvent, NewSession, NewTurn, + SessionRecord, SessionRow, TurnRecord, TurnRow, parse_uuid, timestamp_from_micros, }, schema::{ agent_diagnostic_capture_events, agent_diagnostic_captures, agent_sessions, agent_turns, @@ -616,22 +620,7 @@ pub(super) fn session_timeline( user_id: Uuid, session_pk: Uuid, ) -> Result { - let session = agent_sessions::table - .filter(agent_sessions::id.eq(session_pk.to_string())) - .filter(agent_sessions::user_id.eq(user_id.to_string())) - .select(SessionRow::as_select()) - .first::(connection) - .optional()? - .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))? - .into_record()?; - let turns = agent_turns::table - .filter(agent_turns::session_pk.eq(session.id.to_string())) - .order_by(agent_turns::turn_index.asc()) - .select(TurnRow::as_select()) - .load::(connection)? - .into_iter() - .map(TurnRow::into_record) - .collect::, _>>()?; + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; let mut events = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(session.id.to_string())) .order_by(( @@ -660,6 +649,184 @@ pub(super) fn session_timeline( attachments.remove(&event_pk).unwrap_or_default(), )); } + Ok(assemble_timeline(session, turns, events_by_turn)) +} + +type OutlineRow = ( + String, + String, + String, + i32, + String, + String, + String, + i64, + i64, + i64, + i64, + i64, + i64, + bool, + i64, + i64, + Option, + Option, + Option, + bool, +); + +pub(super) fn session_outline( + connection: &mut SqliteConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result { + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; + // Only fixed-size columns and SQL-computed summaries are selected, so the + // outline stays small no matter how large event text or metadata grows. + let rows = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(session.id.to_string())) + .select(( + llm_usage_events::id, + llm_usage_events::event_id, + llm_usage_events::turn_pk, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + sql::("json_type(metadata, '$.token_usage_estimated') IS 'true'"), + llm_usage_events::observed_at, + sql::("coalesce(length(text), 0)"), + sql::>(&format!( + "CASE WHEN event_type = 'request' THEN substr(text, -{OUTLINE_PREVIEW_CHARS}) \ + ELSE substr(text, 1, {OUTLINE_PREVIEW_CHARS}) END" + )), + sql::>( + "CASE WHEN json_type(metadata, '$.response_id') = 'text' \ + THEN json_extract(metadata, '$.response_id') END", + ), + sql::>( + "CASE WHEN json_type(metadata, '$.previous_response_id') = 'text' \ + THEN json_extract(metadata, '$.previous_response_id') END", + ), + // SQLite text functions stop at the first NUL character, so the + // length and preview above are wrong for text containing NUL. + sql::("coalesce(instr(CAST(text AS BLOB), x'00') > 0, 0)"), + )) + .load::(connection)?; + let nul_text_ids = rows + .iter() + .filter(|row| row.19) + .map(|row| row.0.clone()) + .collect::>(); + let nul_texts = if nul_text_ids.is_empty() { + HashMap::new() + } else { + llm_usage_events::table + .filter(llm_usage_events::id.eq_any(nul_text_ids)) + .select((llm_usage_events::id, llm_usage_events::text)) + .load::<(String, Option)>(connection)? + .into_iter() + .filter_map(|(id, text)| text.map(|text| (id, text))) + .collect::>() + }; + let event_ids = rows.iter().map(|row| row.0.clone()).collect::>(); + let attachment_counts = llm_usage_event_attachments::table + .filter(llm_usage_event_attachments::event_pk.eq_any(event_ids)) + .group_by(llm_usage_event_attachments::event_pk) + .select((llm_usage_event_attachments::event_pk, count_star())) + .load::<(String, i64)>(connection)? + .into_iter() + .collect::>(); + let mut events = rows + .into_iter() + .map(|mut row| { + let attachment_count = attachment_counts.get(&row.0).copied().unwrap_or(0); + if let Some(text) = nul_texts.get(&row.0) { + (row.15, row.16) = text_outline(text, &row.6)?; + } + Ok(UsageEventOutline { + id: parse_uuid(&row.0, "event id")?, + event_id: row.1, + turn_pk: parse_uuid(&row.2, "event turn id")?, + turn_index: row.3, + llm_provider: row.4, + llm_model: row.5, + event_type: row.6, + input_tokens: row.7, + output_tokens: row.8, + cache_read_tokens: row.9, + cache_write_tokens: row.10, + reasoning_tokens: row.11, + total_tokens: row.12, + token_usage_estimated: row.13, + observed_at: timestamp_from_micros(row.14)?, + text_chars: row.15, + preview: row.16, + attachment_count, + ordering_metadata: UsageEventOutline::ordering_metadata(row.17, row.18), + }) + }) + .collect::, AppError>>()?; + UsageEventTimelineOrder::sort(&mut events); + let mut events_by_turn: HashMap> = HashMap::new(); + for event in events { + events_by_turn.entry(event.turn_pk).or_default().push(event); + } + Ok(assemble_timeline(session, turns, events_by_turn)) +} + +/// Computes the outline length and preview in Rust for text SQLite cannot measure. +fn text_outline(text: &str, event_type: &str) -> Result<(i64, Option), AppError> { + let chars = text.chars().count(); + let preview_chars = usize::try_from(OUTLINE_PREVIEW_CHARS) + .map_err(|error| AppError::internal(format!("invalid outline preview size: {error}")))?; + let preview = if event_type == "request" { + text.chars() + .skip(chars.saturating_sub(preview_chars)) + .collect() + } else { + text.chars().take(preview_chars).collect() + }; + let chars = i64::try_from(chars) + .map_err(|error| AppError::internal(format!("invalid event text length: {error}")))?; + Ok((chars, Some(preview))) +} + +fn load_session_turns( + connection: &mut SqliteConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result<(SessionRecord, Vec), AppError> { + let session = agent_sessions::table + .filter(agent_sessions::id.eq(session_pk.to_string())) + .filter(agent_sessions::user_id.eq(user_id.to_string())) + .select(SessionRow::as_select()) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))? + .into_record()?; + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session.id.to_string())) + .order_by(agent_turns::turn_index.asc()) + .select(TurnRow::as_select()) + .load::(connection)? + .into_iter() + .map(TurnRow::into_record) + .collect::, _>>()?; + Ok((session, turns)) +} + +fn assemble_timeline( + session: SessionRecord, + turns: Vec, + mut events_by_turn: HashMap>, +) -> SessionTimelineResponse { let turn_timelines = turns .into_iter() .map(|turn| TurnTimeline { @@ -670,7 +837,7 @@ pub(super) fn session_timeline( events: events_by_turn.remove(&turn.id).unwrap_or_default(), }) .collect(); - Ok(SessionTimelineResponse { + SessionTimelineResponse { session: SessionInfo { session_pk: session.id, user_id: session.user_id, @@ -683,7 +850,33 @@ pub(super) fn session_timeline( metadata: session.metadata, }, turns: turn_timelines, - }) + } +} + +pub(super) fn usage_event( + connection: &mut SqliteConnection, + user_id: Uuid, + event_pk: Uuid, +) -> Result { + let event = llm_usage_events::table + .filter(llm_usage_events::id.eq(event_pk.to_string())) + .filter(llm_usage_events::user_id.eq(user_id.to_string())) + .select(EventRow::as_select()) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("usage event not found: {event_pk}")))? + .into_record()?; + let events = [event]; + let devices = load_device_contexts(connection, &events)?; + let mut attachments = load_attachments(connection, &events)?; + let [event] = events; + let event_pk = event.id; + let device_id = event.device_context_id; + Ok(event_response( + event, + devices.get(&device_id), + attachments.remove(&event_pk).unwrap_or_default(), + )) } pub(super) fn image_attachment( @@ -1108,7 +1301,7 @@ mod tests { use super::{ super::{migrations, schema::llm_usage_events, search}, - ingest_events, + ingest_events, session_outline, }; const OWNER_ID: Uuid = Uuid::from_u128(1); @@ -1194,6 +1387,29 @@ mod tests { assert_eq!(fts_count, 0); } + #[test] + fn outline_measures_text_containing_nul_characters() { + let mut database = database(); + let text = format!("ab\0{}", "c".repeat(300)); + ingest_events(&mut database, &request_with_text(&text), OWNER_ID, 100) + .expect("ingest should succeed"); + let session_pk = llm_usage_events::table + .select(llm_usage_events::session_pk) + .first::(&mut database) + .expect("event should exist"); + let session_pk = Uuid::parse_str(&session_pk).expect("session pk should be a UUID"); + + let outline = + session_outline(&mut database, OWNER_ID, session_pk).expect("outline should load"); + + let event = &outline.turns[0].events[0]; + assert_eq!(event.text_chars, 303); + assert_eq!( + event.preview.as_deref(), + Some(format!("ab\0{}", "c".repeat(197)).as_str()) + ); + } + fn request_with_text(text: &str) -> IngestEventsRequest { serde_json::from_value(json!({ "events": [{ diff --git a/crates/abyss-backend/src/usage/event_order.rs b/crates/abyss-backend/src/usage/event_order.rs index 5969899..64e749c 100644 --- a/crates/abyss-backend/src/usage/event_order.rs +++ b/crates/abyss-backend/src/usage/event_order.rs @@ -44,6 +44,28 @@ impl TimelineEvent for crate::db::models::UsageEvent { } } +impl TimelineEvent for super::UsageEventOutline { + fn turn_index(&self) -> i32 { + self.turn_index + } + + fn metadata(&self) -> &serde_json::Value { + &self.ordering_metadata + } + + fn observed_at(&self) -> DateTime { + self.observed_at + } + + fn event_type(&self) -> &str { + &self.event_type + } + + fn event_id(&self) -> &str { + &self.event_id + } +} + /// Applies the authoritative event order used by owned and shared timelines. pub struct UsageEventTimelineOrder; diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 0d055c2..165e96e 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -11,6 +11,7 @@ pub mod attachments; pub mod diagnostics; pub mod event_order; pub mod persistence; +pub mod trajectory; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -411,11 +412,85 @@ pub struct UsageEventResponse { #[derive(Debug, Serialize)] /// Session metadata and its ordered turn timelines. -pub struct SessionTimelineResponse { +/// +/// `E` is the per-event representation: complete events for the full +/// timeline, or [`UsageEventOutline`] for the content-free outline. +pub struct SessionTimelineResponse { /// Authoritative session information. pub session: SessionInfo, /// Turns ordered by normalized turn index. - pub turns: Vec, + pub turns: Vec>, +} + +/// Session timeline whose events omit text and metadata bodies. +pub type SessionOutlineResponse = SessionTimelineResponse; + +/// Maximum number of characters returned in an outline event preview. +pub const OUTLINE_PREVIEW_CHARS: i64 = 200; + +#[derive(Debug, Serialize)] +/// Content-free representation of one event, sized independently of its text. +/// +/// Full content is loaded separately by event primary key. +pub struct UsageEventOutline { + /// Backend-generated event primary key. + pub id: Uuid, + /// Collector-generated idempotency key. + pub event_id: String, + /// Owning turn, used only to group events. + #[serde(skip)] + pub turn_pk: Uuid, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Canonical LLM provider slug. + pub llm_provider: String, + /// Provider-specific model identifier. + pub llm_model: String, + /// Canonical request or response value. + pub event_type: String, + /// Normalized input-token count. + pub input_tokens: i64, + /// Normalized output-token count. + pub output_tokens: i64, + /// Normalized cache-read-token count. + pub cache_read_tokens: i64, + /// Normalized cache-write-token count. + pub cache_write_tokens: i64, + /// Normalized reasoning-token count. + pub reasoning_tokens: i64, + /// Provider-reported or derived total-token count. + pub total_tokens: i64, + /// Whether the collector estimated token usage. + pub token_usage_estimated: bool, + /// Collector observation timestamp. + pub observed_at: DateTime, + /// Text length in characters; zero when text is absent. + pub text_chars: i64, + /// Up to [`OUTLINE_PREVIEW_CHARS`] characters of text: the tail of a + /// request, where the newest prompt is, or the head of a response. + pub preview: Option, + /// Number of image attachments on the event. + pub attachment_count: i64, + /// Provider response-chain identifiers needed by timeline ordering. + #[serde(skip)] + pub ordering_metadata: Value, +} + +impl UsageEventOutline { + /// Builds the minimal metadata object consumed by timeline ordering. + pub fn ordering_metadata( + response_id: Option, + previous_response_id: Option, + ) -> Value { + let mut metadata = serde_json::Map::new(); + if let Some(value) = response_id { + metadata.insert("response_id".to_owned(), Value::String(value)); + } + if let Some(value) = previous_response_id { + metadata.insert("previous_response_id".to_owned(), Value::String(value)); + } + Value::Object(metadata) + } } #[derive(Debug, Serialize)] @@ -462,7 +537,7 @@ pub fn session_metadata_from_event(metadata: &Value) -> Value { #[derive(Debug, Serialize)] /// One normalized turn and its deterministically ordered events. -pub struct TurnTimeline { +pub struct TurnTimeline { /// Backend turn primary key. pub turn_pk: Uuid, /// Backend-normalized turn number. @@ -472,7 +547,7 @@ pub struct TurnTimeline { /// Latest event observation time in the turn. pub ended_at: Option>, /// Request and response events in authoritative timeline order. - pub events: Vec, + pub events: Vec, } /// Returns the JSON object used when optional metadata is absent. diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs new file mode 100644 index 0000000..18efc94 --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -0,0 +1,422 @@ +//! Merged session trajectory built from accumulated request/response snapshots. +//! +//! Collectors record each provider request verbatim, so every request repeats +//! the conversation so far. The trajectory keeps only what each event adds: +//! new request paragraphs, first-seen tool results, and response text and tool +//! calls, with each tool call paired to its result by call identifier. + +use std::{ + collections::{HashMap, HashSet, hash_map::DefaultHasher}, + hash::{Hash, Hasher}, +}; + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde_json::Value; +use uuid::Uuid; + +use super::{SessionInfo, SessionTimelineResponse, UsageEventResponse}; + +/// Collectors join request fragments with this separator. +const PARAGRAPH_SEPARATOR: &str = "\n\n"; +/// Collectors prefix the system prompt fragment with this marker. +const SYSTEM_PREFIX: &str = "system: "; +/// Shared prefix length above which a changed paragraph counts as grown. +const GROWN_PARAGRAPH_MIN_PREFIX_BYTES: usize = 200; + +#[derive(Debug, Serialize)] +/// Session information and its merged, de-duplicated trajectory items. +pub struct SessionTrajectoryResponse { + /// Authoritative session information. + pub session: SessionInfo, + /// Items in authoritative event order. + pub items: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +/// Kind of one trajectory item. +pub enum TrajectoryItemKind { + /// First request content of a stream, starting with its system prompt. + Context, + /// Request paragraphs not seen earlier in the session. + User, + /// Response text. + Assistant, + /// Response tool call, with its result once one arrives. + ToolCall, + /// Request tool result whose call was not observed. + ToolResult, +} + +#[derive(Debug, Serialize)] +/// One piece of new content contributed by one event. +pub struct TrajectoryItem { + /// Item kind. + pub kind: TrajectoryItemKind, + /// Zero-based request stream, keyed by system prompt. Side requests such + /// as classifiers or subagents form their own streams. + pub stream: usize, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Event that contributed this item. + pub event_pk: Uuid, + /// Observation time of that event. + pub observed_at: DateTime, + /// Model of that event. + pub llm_model: String, + /// Text of context, user, and assistant items. + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// Whether `text` is only the new tail of a paragraph that grew. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub continued: bool, + /// Tool name of a tool call. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Provider call identifier of a tool call or result. + #[serde(skip_serializing_if = "Option::is_none")] + pub call_id: Option, + /// Tool call input. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// Tool result output. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// Event that first carried the tool result. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_event_pk: Option, +} + +impl TrajectoryItem { + fn new(kind: TrajectoryItemKind, stream: usize, event: &UsageEventResponse) -> Self { + Self { + kind, + stream, + turn_index: event.turn_index, + event_pk: event.id, + observed_at: event.observed_at, + llm_model: event.llm_model.clone(), + text: None, + continued: false, + name: None, + call_id: None, + input: None, + output: None, + output_event_pk: None, + } + } +} + +/// Merges an ordered session timeline into its trajectory. +pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { + let mut merger = Merger::default(); + for turn in &timeline.turns { + for event in &turn.events { + if event.event_type == "request" { + merger.request(event); + } else { + merger.response(event); + } + } + } + SessionTrajectoryResponse { + session: timeline.session, + items: merger.items, + } +} + +#[derive(Default)] +struct Merger { + items: Vec, + seen_paragraphs: HashSet, + seen_results: HashSet, + streams: HashMap, + last_paragraphs: HashMap>, + /// Tool call item index by call identifier. + calls: HashMap, + /// Stream of the most recent request; responses belong to it. + current_stream: usize, +} + +impl Merger { + fn request(&mut self, event: &UsageEventResponse) { + let text = event.text.as_deref().unwrap_or_default(); + let paragraphs = text.split(PARAGRAPH_SEPARATOR).collect::>(); + let first = paragraphs.first().copied().unwrap_or_default(); + let stream_key = if first.starts_with(SYSTEM_PREFIX) { + paragraph_key(0, first) + } else { + 0 + }; + let next_stream = self.streams.len(); + let stream = *self.streams.entry(stream_key).or_insert(next_stream); + self.current_stream = stream; + + let previous = self.last_paragraphs.remove(&stream).unwrap_or_default(); + let mut added = Vec::new(); + let mut continued = false; + for (index, paragraph) in paragraphs.iter().enumerate() { + if paragraph.trim().is_empty() + || !self.seen_paragraphs.insert(paragraph_key(index, paragraph)) + { + continue; + } + // A paragraph that grew since the stream's previous request, such + // as an embedded transcript, contributes only its new tail. + let shared = previous + .iter() + .map(|earlier| common_prefix_len(paragraph, earlier)) + .max() + .unwrap_or(0); + if shared >= GROWN_PARAGRAPH_MIN_PREFIX_BYTES { + added.push(¶graph[shared..]); + continued = true; + } else { + added.push(paragraph); + } + } + if !added.is_empty() { + let kind = if added[0].starts_with(SYSTEM_PREFIX) { + TrajectoryItemKind::Context + } else { + TrajectoryItemKind::User + }; + let mut item = TrajectoryItem::new(kind, stream, event); + item.text = Some(added.join(PARAGRAPH_SEPARATOR)); + item.continued = continued; + self.items.push(item); + } + self.last_paragraphs + .insert(stream, paragraphs.into_iter().map(str::to_owned).collect()); + + for segment in content_segments(&event.metadata) { + if segment_str(segment, "type") != Some("tool_result") { + continue; + } + let call_id = segment_str(segment, "call_id").map(str::to_owned); + let output = segment_str(segment, "output").map(str::to_owned); + let result_key = call_id.clone().unwrap_or_else(|| { + format!( + "output:{:x}", + hash_str(output.as_deref().unwrap_or_default()) + ) + }); + if !self.seen_results.insert(result_key) { + continue; + } + if let Some(&index) = call_id.as_ref().and_then(|id| self.calls.get(id)) { + let call = &mut self.items[index]; + call.output = output; + call.output_event_pk = Some(event.id); + } else { + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolResult, stream, event); + item.call_id = call_id; + item.output = output; + item.output_event_pk = Some(event.id); + self.items.push(item); + } + } + } + + fn response(&mut self, event: &UsageEventResponse) { + let stream = self.current_stream; + if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { + let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); + item.text = Some(text.to_owned()); + self.items.push(item); + } + for segment in content_segments(&event.metadata) { + if segment_str(segment, "type") != Some("tool_call") { + continue; + } + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); + item.name = segment_str(segment, "name").map(str::to_owned); + item.call_id = segment_str(segment, "call_id").map(str::to_owned); + item.input = segment_str(segment, "input").map(str::to_owned); + if let Some(call_id) = &item.call_id { + self.calls.insert(call_id.clone(), self.items.len()); + } + self.items.push(item); + } + } +} + +/// Identity of a request paragraph. The system fragment's first line carries +/// per-request noise, such as a billing header, so it is excluded. +fn paragraph_key(index: usize, paragraph: &str) -> u64 { + if index == 0 && paragraph.starts_with(SYSTEM_PREFIX) { + return hash_str(paragraph.split_once('\n').map_or("", |(_, rest)| rest)); + } + hash_str(paragraph) +} + +fn hash_str(value: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() +} + +/// Length in bytes of the shared prefix, ending on a character boundary. +fn common_prefix_len(left: &str, right: &str) -> usize { + let mut len = left + .bytes() + .zip(right.bytes()) + .take_while(|(a, b)| a == b) + .count(); + while !left.is_char_boundary(len) { + len = len.saturating_sub(1); + } + len +} + +fn content_segments(metadata: &Value) -> &[Value] { + metadata + .get("content_segments") + .and_then(Value::as_array) + .map_or(&[], Vec::as_slice) +} + +fn segment_str<'a>(segment: &'a Value, key: &str) -> Option<&'a str> { + segment.get(key).and_then(Value::as_str) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::usage::TurnTimeline; + + #[test] + fn keeps_only_new_content_and_pairs_tool_results() { + let system = "system: header cch=1\nYou are an agent."; + let events = vec![ + event("request", &format!("{system}\n\nfirst prompt"), json!({})), + event( + "response", + "", + json!({"content_segments": [ + {"type": "tool_call", "call_id": "c1", "name": "Bash", "input": "ls"} + ]}), + ), + event( + "request", + "system: header cch=2\nYou are an agent.\n\nfirst prompt", + json!({"content_segments": [ + {"type": "tool_result", "call_id": "c1", "output": "README.md"} + ]}), + ), + event("response", "done", json!({})), + event( + "request", + "system: header cch=3\nYou are an agent.\n\nfirst prompt\n\nsecond prompt", + json!({"content_segments": [ + {"type": "tool_result", "call_id": "c1", "output": "README.md"} + ]}), + ), + ]; + + let items = build(timeline(events)).items; + + let kinds = items.iter().map(|item| item.kind).collect::>(); + assert_eq!( + kinds, + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::ToolCall, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User, + ] + ); + assert_eq!(items[1].output.as_deref(), Some("README.md")); + assert_eq!(items[3].text.as_deref(), Some("second prompt")); + assert!(items.iter().all(|item| item.stream == 0)); + } + + #[test] + fn grown_paragraphs_keep_their_tail_and_side_requests_form_streams() { + let transcript = "x".repeat(300); + let events = vec![ + event("request", "system: main\nMain agent.\n\nhello", json!({})), + event( + "request", + &format!("system: classifier\nJudge.\n\n{transcript}"), + json!({}), + ), + event( + "request", + &format!("system: classifier\nJudge.\n\n{transcript} more"), + json!({}), + ), + ]; + + let items = build(timeline(events)).items; + + assert_eq!(items.len(), 3); + assert_eq!(items[1].stream, 1); + assert_eq!(items[2].stream, 1); + assert_eq!(items[2].text.as_deref(), Some(" more")); + assert!(items[2].continued); + } + + #[test] + fn common_prefix_stops_on_character_boundaries() { + assert_eq!(common_prefix_len("中文a", "中字b"), 3); + } + + fn timeline(events: Vec) -> SessionTimelineResponse { + let now = Utc::now(); + SessionTimelineResponse { + session: SessionInfo { + session_pk: Uuid::nil(), + user_id: Uuid::nil(), + device_context_id: Uuid::nil(), + agent_name: "codex".to_owned(), + agent_version: None, + session_id: "session".to_owned(), + started_at: now, + ended_at: None, + metadata: json!({}), + }, + turns: vec![TurnTimeline { + turn_pk: Uuid::nil(), + turn_index: 1, + started_at: now, + ended_at: None, + events, + }], + } + } + + fn event(event_type: &str, text: &str, metadata: Value) -> UsageEventResponse { + UsageEventResponse { + id: Uuid::now_v7(), + event_id: Uuid::now_v7().to_string(), + user_id: Uuid::nil(), + device_context_id: Uuid::nil(), + host_name: None, + platform: None, + session_pk: Uuid::nil(), + turn_pk: Uuid::nil(), + agent_name: "codex".to_owned(), + agent_version: None, + session_id: "session".to_owned(), + turn_index: 1, + llm_provider: "openai".to_owned(), + llm_model: "gpt-5".to_owned(), + event_type: event_type.to_owned(), + text: (!text.is_empty()).then(|| text.to_owned()), + text_sha256: None, + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 0, + observed_at: Utc::now(), + metadata, + attachments: Vec::new(), + } + } +} diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index 1ca3350..a55bf85 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -10,6 +10,7 @@ import urllib.error import urllib.parse import urllib.request +import uuid from dataclasses import dataclass from typing import Any @@ -271,6 +272,82 @@ def run(base_url: str, token: str, run_id: str) -> None: "session timeline must preserve request-before-response ordering", ) + outline = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{session_pk}/outline", + token=token, + ) + require_status(outline, 200, "session outline") + outline_body = outline.json() + require( + outline_body["session"] == timeline.json()["session"], + "session outline must return the timeline session information", + ) + outline_events = [ + item for turn in outline_body["turns"] for item in turn["events"] + ] + require( + [item["id"] for item in outline_events] + == [item["id"] for item in timeline_events], + "session outline must preserve the timeline event order", + ) + for outline_event, timeline_event in zip(outline_events, timeline_events): + require( + "text" not in outline_event and "metadata" not in outline_event, + "session outline events must omit content bodies", + ) + for field in ("event_type", "llm_model", "total_tokens", "observed_at"): + require( + outline_event[field] == timeline_event[field], + f"session outline {field} must match the timeline event", + ) + require( + outline_event["text_chars"] == len(timeline_event["text"] or ""), + "session outline text length must count characters", + ) + require( + outline_event["attachment_count"] == len(timeline_event["attachments"]), + "session outline must count event attachments", + ) + missing_outline = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{uuid.uuid4()}/outline", + token=token, + ) + require_status(missing_outline, 404, "unknown session outline") + + for timeline_event in timeline_events: + single = request( + base_url, + "GET", + f"/v1/agent-usage/events/{timeline_event['id']}", + token=token, + ) + require_status(single, 200, "single event query") + require( + single.json() == timeline_event, + "single event query must match the timeline event exactly", + ) + require( + IMAGE_BASE64.encode() not in single.body, + "single event responses must not expose attachment content", + ) + missing_event = request( + base_url, + "GET", + f"/v1/agent-usage/events/{uuid.uuid4()}", + token=token, + ) + require_status(missing_event, 404, "unknown single event query") + unauthenticated_event = request( + base_url, + "GET", + f"/v1/agent-usage/events/{timeline_events[0]['id']}", + ) + require_status(unauthenticated_event, 401, "unauthenticated single event query") + search_path = "/v1/agent-usage/search?" + urllib.parse.urlencode( {"q": "blackbox response", "agent_name": "codex"} ) From dd6eef50ac5a000fd49e0b6274c35af55d2ce9ea Mon Sep 17 00:00:00 2001 From: Auceptin Date: Tue, 29 Sep 2026 02:30:35 -0700 Subject: [PATCH 03/30] feat: list sessions by latest activity Add GET /v1/agent-usage/sessions, returning one page of owner-scoped sessions ordered by their latest event, with turn, event, and token counts plus the device host and platform. Sessions can be filtered by an activity window and Agent name, with Claude aliases matched together. Implemented for both the SQLite and PostgreSQL backends. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/api.rs | 12 + crates/abyss-backend/src/storage/mod.rs | 11 +- crates/abyss-backend/src/storage/postgres.rs | 16 +- .../src/storage/postgres/session_list.rs | 136 +++++++++ .../abyss-backend/src/storage/sqlite/mod.rs | 16 +- .../src/storage/sqlite/session_list.rs | 279 ++++++++++++++++++ crates/abyss-backend/src/usage/mod.rs | 2 + .../abyss-backend/src/usage/session_list.rs | 175 +++++++++++ 8 files changed, 644 insertions(+), 3 deletions(-) create mode 100644 crates/abyss-backend/src/storage/postgres/session_list.rs create mode 100644 crates/abyss-backend/src/storage/sqlite/session_list.rs create mode 100644 crates/abyss-backend/src/usage/session_list.rs diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index 59c3213..6d36468 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -25,6 +25,7 @@ use crate::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, SessionOutlineResponse, SessionTimelineResponse, SummaryFields, SummaryQuery, TokenUsageSummaryResponse, UsageEventResponse, + session_list::{SessionListQuery, SessionListResponse}, trajectory::{self, SessionTrajectoryResponse}, }, }; @@ -70,6 +71,7 @@ pub fn router(state: AppState) -> Router { ) .route("/v1/agent-usage/summary", get(usage_summary)) .route("/v1/agent-usage/search", get(session_search)) + .route("/v1/agent-usage/sessions", get(list_sessions)) .route( "/v1/agent-usage/sessions/{session_pk}", get(session_timeline), @@ -153,6 +155,16 @@ async fn session_search( Ok(Json(state.storage.session_search(user_id, query).await?)) } +async fn list_sessions( + State(state): State, + headers: HeaderMap, + Query(query): Query, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + let query = query.validate()?; + Ok(Json(state.storage.list_sessions(user_id, query).await?)) +} + async fn session_timeline( State(state): State, headers: HeaderMap, diff --git a/crates/abyss-backend/src/storage/mod.rs b/crates/abyss-backend/src/storage/mod.rs index ebf0d8e..3f846cc 100644 --- a/crates/abyss-backend/src/storage/mod.rs +++ b/crates/abyss-backend/src/storage/mod.rs @@ -21,7 +21,9 @@ use crate::{ usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, - UsageEventResponse, attachments::StoredImageAttachment, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -78,6 +80,13 @@ pub trait StorageBackend: Send + Sync { session_pk: Uuid, ) -> BoxedFuture<'_, Result>; + /// Lists one page of owner-scoped sessions, most recently active first. + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result>; + /// Loads authorized image bytes by attachment identifier. fn image_attachment( &self, diff --git a/crates/abyss-backend/src/storage/postgres.rs b/crates/abyss-backend/src/storage/postgres.rs index f267bd8..04caabc 100644 --- a/crates/abyss-backend/src/storage/postgres.rs +++ b/crates/abyss-backend/src/storage/postgres.rs @@ -2,6 +2,7 @@ mod repository; mod search; +mod session_list; use std::time::Duration; @@ -18,7 +19,9 @@ use crate::{ usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, - UsageEventResponse, attachments::StoredImageAttachment, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -165,6 +168,17 @@ impl StorageBackend for PostgresEsBackend { }) } + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| session_list::list_sessions(connection, user_id, &query)) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/postgres/session_list.rs b/crates/abyss-backend/src/storage/postgres/session_list.rs new file mode 100644 index 0000000..8b66595 --- /dev/null +++ b/crates/abyss-backend/src/storage/postgres/session_list.rs @@ -0,0 +1,136 @@ +//! PostgreSQL session index ordered by latest activity. + +use chrono::{DateTime, Utc}; +use diesel::{ + PgConnection, QueryableByName, RunQueryDsl, sql_query, + sql_types::{Array, BigInt, Bool, Nullable, Text, Timestamptz, Uuid as SqlUuid}, +}; +use uuid::Uuid; + +use crate::{ + error::AppError, + usage::session_list::{SessionListItem, SessionListResponse, ValidatedSessionListQuery}, +}; + +/// Per-session activity with the filters that apply before paging. +/// +/// Bind order: `$1` owner, `$2`/`$3` agent filter flag and names, `$4`/`$5` +/// lower activity bound, `$6`/`$7` upper start bound. +const ACTIVITY_CTE: &str = "\ + WITH activity AS ( \ + SELECT \ + s.id AS session_pk, \ + s.session_id AS session_id, \ + s.agent_name AS agent_name, \ + s.agent_version AS agent_version, \ + d.host_name AS host_name, \ + d.platform AS platform, \ + s.started_at AS started_at, \ + s.ended_at AS ended_at, \ + COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ + (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id)::bigint AS turns, \ + COUNT(e.id)::bigint AS events, \ + COALESCE(SUM(e.total_tokens), 0)::bigint AS total_tokens \ + FROM agent_sessions s \ + JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ + WHERE s.user_id = $1 \ + AND ($2 = false OR s.agent_name = ANY($3)) \ + GROUP BY s.id, d.host_name, d.platform \ + ), \ + filtered AS ( \ + SELECT * FROM activity \ + WHERE ($4 = false OR last_activity_at >= $5) \ + AND ($6 = false OR started_at < $7) \ + ) "; + +#[derive(QueryableByName)] +struct SessionListRow { + #[diesel(sql_type = SqlUuid)] + session_pk: Uuid, + #[diesel(sql_type = Text)] + session_id: String, + #[diesel(sql_type = Text)] + agent_name: String, + #[diesel(sql_type = Nullable)] + agent_version: Option, + #[diesel(sql_type = Text)] + host_name: String, + #[diesel(sql_type = Text)] + platform: String, + #[diesel(sql_type = Timestamptz)] + started_at: DateTime, + #[diesel(sql_type = Nullable)] + ended_at: Option>, + #[diesel(sql_type = Timestamptz)] + last_activity_at: DateTime, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + events: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, +} + +#[derive(QueryableByName)] +struct CountRow { + #[diesel(sql_type = BigInt)] + total: i64, +} + +pub(super) fn list_sessions( + connection: &mut PgConnection, + user_id: Uuid, + query: &ValidatedSessionListQuery, +) -> Result { + let has_agents = !query.agent_names.is_empty(); + let from = query.from.unwrap_or(DateTime::UNIX_EPOCH); + let to = query.to.unwrap_or(DateTime::UNIX_EPOCH); + + let total = sql_query(format!( + "{ACTIVITY_CTE} SELECT COUNT(*)::bigint AS total FROM filtered" + )) + .bind::(user_id) + .bind::(has_agents) + .bind::, _>(&query.agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .get_result::(connection)? + .total; + + let rows = sql_query(format!( + "{ACTIVITY_CTE} SELECT * FROM filtered \ + ORDER BY last_activity_at DESC, session_pk DESC LIMIT $8 OFFSET $9" + )) + .bind::(user_id) + .bind::(has_agents) + .bind::, _>(&query.agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(i64::from(query.page_size)) + .bind::(query.offset) + .load::(connection)?; + + let items = rows + .into_iter() + .map(|row| SessionListItem { + session_pk: row.session_pk, + session_id: row.session_id, + agent_name: row.agent_name, + agent_version: row.agent_version, + host_name: row.host_name, + platform: row.platform, + started_at: row.started_at, + ended_at: row.ended_at, + last_activity_at: row.last_activity_at, + turns: row.turns, + events: row.events, + total_tokens: row.total_tokens, + }) + .collect(); + Ok(SessionListResponse::new(query, total, items)) +} diff --git a/crates/abyss-backend/src/storage/sqlite/mod.rs b/crates/abyss-backend/src/storage/sqlite/mod.rs index 431b28c..9cf4669 100644 --- a/crates/abyss-backend/src/storage/sqlite/mod.rs +++ b/crates/abyss-backend/src/storage/sqlite/mod.rs @@ -6,6 +6,7 @@ mod models; mod repository; mod schema; mod search; +mod session_list; use diesel::{QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star}; use uuid::Uuid; @@ -18,7 +19,9 @@ use crate::{ usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, - UsageEventResponse, attachments::StoredImageAttachment, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -149,6 +152,17 @@ impl StorageBackend for SqliteFtsBackend { }) } + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| session_list::list_sessions(connection, user_id, &query)) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs new file mode 100644 index 0000000..548cbd9 --- /dev/null +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -0,0 +1,279 @@ +//! SQLite session index ordered by latest activity. + +use diesel::{ + QueryableByName, RunQueryDsl, SqliteConnection, sql_query, + sql_types::{BigInt, Bool, Nullable, Text}, +}; +use uuid::Uuid; + +use crate::{ + error::AppError, + usage::session_list::{SessionListItem, SessionListResponse, ValidatedSessionListQuery}, +}; + +use super::{ + models::{parse_uuid, timestamp_from_micros}, + repository::timestamp_to_micros, +}; + +/// Per-session activity with the filters that apply before paging. +/// +/// Bind order: `?1` owner, `?2`/`?3` agent filter flag and JSON name list, +/// `?4`/`?5` lower activity bound, `?6`/`?7` upper start bound. +const ACTIVITY_CTE: &str = "\ + WITH activity AS ( \ + SELECT \ + s.id AS session_pk, \ + s.session_id AS session_id, \ + s.agent_name AS agent_name, \ + s.agent_version AS agent_version, \ + d.host_name AS host_name, \ + d.platform AS platform, \ + s.started_at AS started_at, \ + s.ended_at AS ended_at, \ + COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ + (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id) AS turns, \ + COUNT(e.id) AS events, \ + COALESCE(SUM(e.total_tokens), 0) AS total_tokens \ + FROM agent_sessions s \ + JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ + WHERE s.user_id = ?1 \ + AND (?2 = 0 OR s.agent_name IN (SELECT value FROM json_each(?3))) \ + GROUP BY s.id \ + ), \ + filtered AS ( \ + SELECT * FROM activity \ + WHERE (?4 = 0 OR last_activity_at >= ?5) \ + AND (?6 = 0 OR started_at < ?7) \ + ) "; + +#[derive(QueryableByName)] +struct SessionListRow { + #[diesel(sql_type = Text)] + session_pk: String, + #[diesel(sql_type = Text)] + session_id: String, + #[diesel(sql_type = Text)] + agent_name: String, + #[diesel(sql_type = Nullable)] + agent_version: Option, + #[diesel(sql_type = Text)] + host_name: String, + #[diesel(sql_type = Text)] + platform: String, + #[diesel(sql_type = BigInt)] + started_at: i64, + #[diesel(sql_type = Nullable)] + ended_at: Option, + #[diesel(sql_type = BigInt)] + last_activity_at: i64, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + events: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, +} + +#[derive(QueryableByName)] +struct CountRow { + #[diesel(sql_type = BigInt)] + total: i64, +} + +pub(super) fn list_sessions( + connection: &mut SqliteConnection, + user_id: Uuid, + query: &ValidatedSessionListQuery, +) -> Result { + let agent_names = serde_json::to_string(&query.agent_names) + .map_err(|error| AppError::internal(format!("serialize agent filter: {error}")))?; + let has_agents = !query.agent_names.is_empty(); + let from = query.from.map_or(0, timestamp_to_micros); + let to = query.to.map_or(0, timestamp_to_micros); + + let total = sql_query(format!( + "{ACTIVITY_CTE} SELECT COUNT(*) AS total FROM filtered" + )) + .bind::(user_id.to_string()) + .bind::(has_agents) + .bind::(&agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .get_result::(connection)? + .total; + + let rows = sql_query(format!( + "{ACTIVITY_CTE} SELECT * FROM filtered \ + ORDER BY last_activity_at DESC, session_pk DESC LIMIT ?8 OFFSET ?9" + )) + .bind::(user_id.to_string()) + .bind::(has_agents) + .bind::(&agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(i64::from(query.page_size)) + .bind::(query.offset) + .load::(connection)?; + + let items = rows + .into_iter() + .map(|row| { + Ok(SessionListItem { + session_pk: parse_uuid(&row.session_pk, "session id")?, + session_id: row.session_id, + agent_name: row.agent_name, + agent_version: row.agent_version, + host_name: row.host_name, + platform: row.platform, + started_at: timestamp_from_micros(row.started_at)?, + ended_at: row.ended_at.map(timestamp_from_micros).transpose()?, + last_activity_at: timestamp_from_micros(row.last_activity_at)?, + turns: row.turns, + events: row.events, + total_tokens: row.total_tokens, + }) + }) + .collect::, AppError>>()?; + Ok(SessionListResponse::new(query, total, items)) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use diesel::{Connection, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use crate::usage::{IngestEventsRequest, session_list::SessionListQuery}; + + use super::{super::migrations, super::repository::ingest_events, list_sessions}; + + const OWNER_ID: Uuid = Uuid::from_u128(1); + + fn database() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + database + } + + fn event(id: &str, session: &str, agent: &str, observed_at: &str, turn: i32) -> Value { + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": session, + "turn_index": turn, + "llm": {"provider": "OpenAI", "model": "gpt-5"}, + "event_type": "response", + "text": "reply", + "token_usage": {"output_tokens": 10}, + "metadata": {"response_id": id} + }) + } + + fn seeded() -> SqliteConnection { + let mut database = database(); + let request: IngestEventsRequest = serde_json::from_value(json!({ + "events": [ + event("a1", "alpha", "Codex", "2026-09-01T00:00:00Z", 1), + event("a2", "alpha", "Codex", "2026-09-03T00:00:00Z", 2), + event("b1", "beta", "claude-desktop", "2026-09-02T00:00:00Z", 1), + event("c1", "gamma", "Codex", "2026-08-01T00:00:00Z", 1), + ] + })) + .expect("request should deserialize"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn list(database: &mut SqliteConnection, query: SessionListQuery) -> Vec<(String, i64, i64)> { + let query = query.validate().expect("query should validate"); + list_sessions(database, OWNER_ID, &query) + .expect("sessions should list") + .items + .into_iter() + .map(|item| (item.session_id, item.turns, item.events)) + .collect() + } + + #[test] + fn lists_sessions_by_latest_activity_with_counts() { + let mut database = seeded(); + let query = SessionListQuery::default().validate().expect("valid"); + let response = list_sessions(&mut database, OWNER_ID, &query).expect("should list"); + + assert_eq!(response.total_sessions, 3); + assert!(!response.has_more); + let alpha = &response.items[0]; + assert_eq!(alpha.session_id, "alpha"); + assert_eq!((alpha.turns, alpha.events, alpha.total_tokens), (2, 2, 20)); + assert_eq!(alpha.host_name, "alice-mbp"); + assert_eq!( + alpha.last_activity_at, + Utc.with_ymd_and_hms(2026, 9, 3, 0, 0, 0).unwrap() + ); + let order: Vec<_> = response + .items + .iter() + .map(|item| item.session_id.as_str()) + .collect(); + assert_eq!(order, ["alpha", "beta", "gamma"]); + } + + #[test] + fn filters_by_agent_alias_and_activity_window() { + let mut database = seeded(); + let claude = SessionListQuery { + agent_name: Some("Claude Code".to_owned()), + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, claude), [("beta".to_owned(), 1, 1)]); + + // gamma went quiet before the window; alpha is still active inside it. + let window = SessionListQuery { + from: Some(Utc.with_ymd_and_hms(2026, 8, 15, 0, 0, 0).unwrap()), + to: Some(Utc.with_ymd_and_hms(2026, 9, 2, 12, 0, 0).unwrap()), + ..SessionListQuery::default() + }; + let ids: Vec<_> = list(&mut database, window) + .into_iter() + .map(|row| row.0) + .collect(); + assert_eq!(ids, ["alpha", "beta"]); + } + + #[test] + fn pages_through_sessions() { + let mut database = seeded(); + let first = SessionListQuery { + page_size: Some(2), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + let page = list_sessions(&mut database, OWNER_ID, &first).expect("should list"); + assert_eq!( + (page.items.len(), page.total_sessions, page.has_more), + (2, 3, true) + ); + + let second = SessionListQuery { + page: Some(2), + page_size: Some(2), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + let page = list_sessions(&mut database, OWNER_ID, &second).expect("should list"); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].session_id, "gamma"); + assert!(!page.has_more); + } +} diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index e833149..a2b1cd7 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -11,6 +11,8 @@ pub mod attachments; pub mod diagnostics; pub mod event_order; pub mod persistence; +/// Owner-scoped session index contracts. +pub mod session_list; pub mod trajectory; use chrono::{DateTime, Utc}; diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs new file mode 100644 index 0000000..45fceac --- /dev/null +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -0,0 +1,175 @@ +//! Owner-scoped session index: one page of sessions ordered by latest activity. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{ + error::AppError, + usage::persistence::{agent_name_filter_values, normalized_optional}, +}; + +const DEFAULT_PAGE_SIZE: u32 = 20; +const MAX_PAGE_SIZE: u32 = 100; + +/// Query parameters for `GET /v1/agent-usage/sessions`. +#[derive(Debug, Default, Deserialize)] +pub struct SessionListQuery { + /// Keeps sessions whose latest activity is at or after this instant. + pub from: Option>, + /// Keeps sessions that started before this instant. + pub to: Option>, + /// Agent-name filter; Claude aliases match together. + pub agent_name: Option, + /// 1-based page number. + pub page: Option, + /// Sessions per page, at most 100. + pub page_size: Option, +} + +/// Session list query with normalized filters and a resolved page window. +#[derive(Debug)] +pub struct ValidatedSessionListQuery { + pub from: Option>, + pub to: Option>, + /// Stored agent names to match, or empty for every Agent. + pub agent_names: Vec, + pub page: u32, + pub page_size: u32, + pub offset: i64, +} + +impl SessionListQuery { + pub fn validate(self) -> Result { + if let (Some(from), Some(to)) = (self.from, self.to) + && from >= to + { + return Err(AppError::validation( + "from must be earlier than to".to_owned(), + )); + } + let page = self.page.unwrap_or(1); + if page == 0 { + return Err(AppError::validation("page must be at least 1".to_owned())); + } + let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); + if page_size == 0 || page_size > MAX_PAGE_SIZE { + return Err(AppError::validation(format!( + "page_size must be between 1 and {MAX_PAGE_SIZE}" + ))); + } + let offset = i64::from(page.saturating_sub(1)).saturating_mul(i64::from(page_size)); + Ok(ValidatedSessionListQuery { + from: self.from, + to: self.to, + agent_names: normalized_optional(self.agent_name.as_deref()) + .map(|value| agent_name_filter_values(&value)) + .unwrap_or_default(), + page, + page_size, + offset, + }) + } +} + +/// One page of sessions, newest activity first. +#[derive(Debug, Serialize)] +pub struct SessionListResponse { + /// Sessions matching the filters across all pages. + pub total_sessions: i64, + pub page: u32, + pub page_size: u32, + pub has_more: bool, + pub items: Vec, +} + +/// Summary of one session for the session index. +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct SessionListItem { + pub session_pk: Uuid, + pub session_id: String, + pub agent_name: String, + pub agent_version: Option, + pub host_name: String, + pub platform: String, + pub started_at: DateTime, + pub ended_at: Option>, + /// Latest event observation, or the session start when it has no events. + pub last_activity_at: DateTime, + pub turns: i64, + pub events: i64, + pub total_tokens: i64, +} + +impl SessionListResponse { + pub fn new( + query: &ValidatedSessionListQuery, + total_sessions: i64, + items: Vec, + ) -> Self { + let shown = query + .offset + .saturating_add(i64::try_from(items.len()).unwrap_or(i64::MAX)); + Self { + total_sessions, + page: query.page, + page_size: query.page_size, + has_more: shown < total_sessions, + items, + } + } +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + + #[test] + fn defaults_to_the_first_page_of_every_agent() { + let query = SessionListQuery::default().validate().expect("valid"); + assert_eq!((query.page, query.page_size, query.offset), (1, 20, 0)); + assert!(query.agent_names.is_empty()); + } + + #[test] + fn expands_claude_aliases_and_computes_the_offset() { + let query = SessionListQuery { + agent_name: Some(" Claude ".to_owned()), + page: Some(3), + page_size: Some(25), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert_eq!(query.offset, 50); + assert_eq!( + query.agent_names, + ["claude-code", "claude", "claude-desktop"] + ); + } + + #[test] + fn rejects_bad_windows_and_page_sizes() { + let at = Utc.with_ymd_and_hms(2026, 9, 1, 0, 0, 0).unwrap(); + let reversed = SessionListQuery { + from: Some(at), + to: Some(at), + ..SessionListQuery::default() + }; + assert!(reversed.validate().is_err()); + for page_size in [0, 101] { + let query = SessionListQuery { + page_size: Some(page_size), + ..SessionListQuery::default() + }; + assert!(query.validate().is_err()); + } + let zero_page = SessionListQuery { + page: Some(0), + ..SessionListQuery::default() + }; + assert!(zero_page.validate().is_err()); + } +} From 8b90475d604b8ea13d89e3a52b757bf587f0e717 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Tue, 29 Sep 2026 03:21:02 -0700 Subject: [PATCH 04/30] feat: filter the session list by provider and model used Keep sessions with at least one event from the given LLM provider or model, so the session index can answer which sessions used a provider or model. Co-Authored-By: Claude Opus 5.5 --- .../src/storage/postgres/session_list.rs | 21 ++++++- .../src/storage/sqlite/session_list.rs | 62 +++++++++++++++++-- .../abyss-backend/src/usage/session_list.rs | 33 +++++++++- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/crates/abyss-backend/src/storage/postgres/session_list.rs b/crates/abyss-backend/src/storage/postgres/session_list.rs index 8b66595..d9616d7 100644 --- a/crates/abyss-backend/src/storage/postgres/session_list.rs +++ b/crates/abyss-backend/src/storage/postgres/session_list.rs @@ -15,7 +15,8 @@ use crate::{ /// Per-session activity with the filters that apply before paging. /// /// Bind order: `$1` owner, `$2`/`$3` agent filter flag and names, `$4`/`$5` -/// lower activity bound, `$6`/`$7` upper start bound. +/// lower activity bound, `$6`/`$7` upper start bound, `$8`/`$9` provider and +/// `$10`/`$11` model a session must have used. const ACTIVITY_CTE: &str = "\ WITH activity AS ( \ SELECT \ @@ -36,6 +37,12 @@ const ACTIVITY_CTE: &str = "\ LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ WHERE s.user_id = $1 \ AND ($2 = false OR s.agent_name = ANY($3)) \ + AND ($8 = false OR EXISTS ( \ + SELECT 1 FROM llm_usage_events p \ + WHERE p.session_pk = s.id AND p.llm_provider = $9)) \ + AND ($10 = false OR EXISTS ( \ + SELECT 1 FROM llm_usage_events m \ + WHERE m.session_pk = s.id AND m.llm_model = $11)) \ GROUP BY s.id, d.host_name, d.platform \ ), \ filtered AS ( \ @@ -86,6 +93,8 @@ pub(super) fn list_sessions( let has_agents = !query.agent_names.is_empty(); let from = query.from.unwrap_or(DateTime::UNIX_EPOCH); let to = query.to.unwrap_or(DateTime::UNIX_EPOCH); + let provider = query.llm_provider.as_deref().unwrap_or_default(); + let model = query.llm_model.as_deref().unwrap_or_default(); let total = sql_query(format!( "{ACTIVITY_CTE} SELECT COUNT(*)::bigint AS total FROM filtered" @@ -97,12 +106,16 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) + .bind::(query.llm_provider.is_some()) + .bind::(provider) + .bind::(query.llm_model.is_some()) + .bind::(model) .get_result::(connection)? .total; let rows = sql_query(format!( "{ACTIVITY_CTE} SELECT * FROM filtered \ - ORDER BY last_activity_at DESC, session_pk DESC LIMIT $8 OFFSET $9" + ORDER BY last_activity_at DESC, session_pk DESC LIMIT $12 OFFSET $13" )) .bind::(user_id) .bind::(has_agents) @@ -111,6 +124,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) + .bind::(query.llm_provider.is_some()) + .bind::(provider) + .bind::(query.llm_model.is_some()) + .bind::(model) .bind::(i64::from(query.page_size)) .bind::(query.offset) .load::(connection)?; diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index 548cbd9..4481cb3 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -19,7 +19,8 @@ use super::{ /// Per-session activity with the filters that apply before paging. /// /// Bind order: `?1` owner, `?2`/`?3` agent filter flag and JSON name list, -/// `?4`/`?5` lower activity bound, `?6`/`?7` upper start bound. +/// `?4`/`?5` lower activity bound, `?6`/`?7` upper start bound, `?8`/`?9` +/// provider and `?10`/`?11` model a session must have used. const ACTIVITY_CTE: &str = "\ WITH activity AS ( \ SELECT \ @@ -40,6 +41,12 @@ const ACTIVITY_CTE: &str = "\ LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ WHERE s.user_id = ?1 \ AND (?2 = 0 OR s.agent_name IN (SELECT value FROM json_each(?3))) \ + AND (?8 = 0 OR EXISTS ( \ + SELECT 1 FROM llm_usage_events p \ + WHERE p.session_pk = s.id AND p.llm_provider = ?9)) \ + AND (?10 = 0 OR EXISTS ( \ + SELECT 1 FROM llm_usage_events m \ + WHERE m.session_pk = s.id AND m.llm_model = ?11)) \ GROUP BY s.id \ ), \ filtered AS ( \ @@ -92,6 +99,8 @@ pub(super) fn list_sessions( let has_agents = !query.agent_names.is_empty(); let from = query.from.map_or(0, timestamp_to_micros); let to = query.to.map_or(0, timestamp_to_micros); + let provider = query.llm_provider.as_deref().unwrap_or_default(); + let model = query.llm_model.as_deref().unwrap_or_default(); let total = sql_query(format!( "{ACTIVITY_CTE} SELECT COUNT(*) AS total FROM filtered" @@ -103,12 +112,16 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) + .bind::(query.llm_provider.is_some()) + .bind::(provider) + .bind::(query.llm_model.is_some()) + .bind::(model) .get_result::(connection)? .total; let rows = sql_query(format!( "{ACTIVITY_CTE} SELECT * FROM filtered \ - ORDER BY last_activity_at DESC, session_pk DESC LIMIT ?8 OFFSET ?9" + ORDER BY last_activity_at DESC, session_pk DESC LIMIT ?12 OFFSET ?13" )) .bind::(user_id.to_string()) .bind::(has_agents) @@ -117,6 +130,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) + .bind::(query.llm_provider.is_some()) + .bind::(provider) + .bind::(query.llm_model.is_some()) + .bind::(model) .bind::(i64::from(query.page_size)) .bind::(query.offset) .load::(connection)?; @@ -163,6 +180,17 @@ mod tests { } fn event(id: &str, session: &str, agent: &str, observed_at: &str, turn: i32) -> Value { + event_with_model(id, session, agent, observed_at, turn, "gpt-5") + } + + fn event_with_model( + id: &str, + session: &str, + agent: &str, + observed_at: &str, + turn: i32, + model: &str, + ) -> Value { json!({ "event_id": id, "observed_at": observed_at, @@ -170,7 +198,7 @@ mod tests { "agent": {"name": agent}, "session_id": session, "turn_index": turn, - "llm": {"provider": "OpenAI", "model": "gpt-5"}, + "llm": {"provider": "OpenAI", "model": model}, "event_type": "response", "text": "reply", "token_usage": {"output_tokens": 10}, @@ -183,7 +211,7 @@ mod tests { let request: IngestEventsRequest = serde_json::from_value(json!({ "events": [ event("a1", "alpha", "Codex", "2026-09-01T00:00:00Z", 1), - event("a2", "alpha", "Codex", "2026-09-03T00:00:00Z", 2), + event_with_model("a2", "alpha", "Codex", "2026-09-03T00:00:00Z", 2, "gpt-5-mini"), event("b1", "beta", "claude-desktop", "2026-09-02T00:00:00Z", 1), event("c1", "gamma", "Codex", "2026-08-01T00:00:00Z", 1), ] @@ -249,6 +277,32 @@ mod tests { assert_eq!(ids, ["alpha", "beta"]); } + #[test] + fn filters_by_a_model_or_provider_the_session_used() { + let mut database = seeded(); + let mini = SessionListQuery { + llm_model: Some("gpt-5-mini".to_owned()), + ..SessionListQuery::default() + }; + let ids: Vec<_> = list(&mut database, mini) + .into_iter() + .map(|row| row.0) + .collect(); + assert_eq!(ids, ["alpha"]); + + let openai = SessionListQuery { + llm_provider: Some("OpenAI".to_owned()), + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, openai).len(), 3); + + let other = SessionListQuery { + llm_provider: Some("anthropic".to_owned()), + ..SessionListQuery::default() + }; + assert!(list(&mut database, other).is_empty()); + } + #[test] fn pages_through_sessions() { let mut database = seeded(); diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index 45fceac..dff6cb1 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::{ error::AppError, - usage::persistence::{agent_name_filter_values, normalized_optional}, + usage::persistence::{agent_name_filter_values, normalize_slug, normalized_optional}, }; const DEFAULT_PAGE_SIZE: u32 = 20; @@ -21,6 +21,10 @@ pub struct SessionListQuery { pub to: Option>, /// Agent-name filter; Claude aliases match together. pub agent_name: Option, + /// Keeps sessions with at least one event from this LLM provider. + pub llm_provider: Option, + /// Keeps sessions with at least one event from this LLM model. + pub llm_model: Option, /// 1-based page number. pub page: Option, /// Sessions per page, at most 100. @@ -34,6 +38,10 @@ pub struct ValidatedSessionListQuery { pub to: Option>, /// Stored agent names to match, or empty for every Agent. pub agent_names: Vec, + /// Normalized provider slug a session must have used. + pub llm_provider: Option, + /// Exact model a session must have used. + pub llm_model: Option, pub page: u32, pub page_size: u32, pub offset: i64, @@ -65,6 +73,9 @@ impl SessionListQuery { agent_names: normalized_optional(self.agent_name.as_deref()) .map(|value| agent_name_filter_values(&value)) .unwrap_or_default(), + llm_provider: normalized_optional(self.llm_provider.as_deref()) + .map(|value| normalize_slug(&value)), + llm_model: normalized_optional(self.llm_model.as_deref()), page, page_size, offset, @@ -150,6 +161,26 @@ mod tests { ); } + #[test] + fn normalizes_provider_and_keeps_the_model_verbatim() { + let query = SessionListQuery { + llm_provider: Some(" OpenAI ".to_owned()), + llm_model: Some(" gpt-5 ".to_owned()), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert_eq!(query.llm_provider.as_deref(), Some("openai")); + assert_eq!(query.llm_model.as_deref(), Some("gpt-5")); + let blank = SessionListQuery { + llm_model: Some(" ".to_owned()), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert!(blank.llm_model.is_none()); + } + #[test] fn rejects_bad_windows_and_page_sizes() { let at = Utc.with_ymd_and_hms(2026, 9, 1, 0, 0, 0).unwrap(); From 14a13a24544a6da1146d88974fcae9b5c1de61a1 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 04:06:37 -0700 Subject: [PATCH 05/30] feat: name sessions from captured title-generation calls Claude Code and Codex ask a model to name each session and return {"title": ...}. Recognize those calls at ingestion by their official naming instruction plus a response made only of agreeing title objects, store the result in session_titles, and expose title/purpose on session info and the session list. Codex naming threads are marked purpose=title_generation. A one-time backfill covers events stored before capture existed. Co-Authored-By: Claude Opus 5.5 --- .../20260930000000_session_titles/down.sql | 2 + .../20260930000000_session_titles/up.sql | 18 + .../20260930000000_session_titles/down.sql | 2 + .../20260930000000_session_titles/up.sql | 18 + crates/abyss-backend/src/db/schema.rs | 26 + crates/abyss-backend/src/storage/postgres.rs | 3 + .../src/storage/postgres/repository.rs | 75 ++- .../src/storage/postgres/session_list.rs | 13 +- .../src/storage/postgres/session_titles.rs | 239 +++++++++ .../abyss-backend/src/storage/sqlite/mod.rs | 2 + .../src/storage/sqlite/repository.rs | 40 +- .../src/storage/sqlite/schema.rs | 26 + .../src/storage/sqlite/session_list.rs | 11 +- .../src/storage/sqlite/session_titles.rs | 488 ++++++++++++++++++ crates/abyss-backend/src/usage/mod.rs | 6 + .../abyss-backend/src/usage/session_list.rs | 5 + .../abyss-backend/src/usage/session_title.rs | 130 +++++ crates/abyss-backend/src/usage/trajectory.rs | 2 + 18 files changed, 1080 insertions(+), 26 deletions(-) create mode 100644 crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql create mode 100644 crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql create mode 100644 crates/abyss-backend/migrations/20260930000000_session_titles/down.sql create mode 100644 crates/abyss-backend/migrations/20260930000000_session_titles/up.sql create mode 100644 crates/abyss-backend/src/storage/postgres/session_titles.rs create mode 100644 crates/abyss-backend/src/storage/sqlite/session_titles.rs create mode 100644 crates/abyss-backend/src/usage/session_title.rs diff --git a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql new file mode 100644 index 0000000..76cec66 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql @@ -0,0 +1,2 @@ +DROP TABLE schema_backfills; +DROP TABLE session_titles; diff --git a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql new file mode 100644 index 0000000..ff6ea06 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql @@ -0,0 +1,18 @@ +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads. +CREATE TABLE session_titles ( + session_pk TEXT PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title TEXT, + title_event_pk TEXT REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at INTEGER, + purpose TEXT, + updated_at INTEGER NOT NULL +) STRICT; + +-- One-time data backfills that run in application code after migrations. +CREATE TABLE schema_backfills ( + name TEXT PRIMARY KEY, + completed_at INTEGER NOT NULL +) STRICT; diff --git a/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql b/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql new file mode 100644 index 0000000..76cec66 --- /dev/null +++ b/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql @@ -0,0 +1,2 @@ +DROP TABLE schema_backfills; +DROP TABLE session_titles; diff --git a/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql b/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql new file mode 100644 index 0000000..51d64c6 --- /dev/null +++ b/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql @@ -0,0 +1,18 @@ +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads. +CREATE TABLE session_titles ( + session_pk uuid PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title text, + title_event_pk uuid REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at timestamptz, + purpose text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- One-time data backfills that run in application code after migrations. +CREATE TABLE schema_backfills ( + name text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() +); diff --git a/crates/abyss-backend/src/db/schema.rs b/crates/abyss-backend/src/db/schema.rs index 2e40b28..5b8cedd 100644 --- a/crates/abyss-backend/src/db/schema.rs +++ b/crates/abyss-backend/src/db/schema.rs @@ -152,6 +152,29 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::{Nullable, Text, Timestamptz, Uuid}; + + session_titles (session_pk) { + session_pk -> Uuid, + user_id -> Uuid, + title -> Nullable, + title_event_pk -> Nullable, + title_observed_at -> Nullable, + purpose -> Nullable, + updated_at -> Timestamptz, + } +} + +diesel::table! { + use diesel::sql_types::{Text, Timestamptz}; + + schema_backfills (name) { + name -> Text, + completed_at -> Timestamptz, + } +} + diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -168,6 +191,7 @@ diesel::joinable!(agent_diagnostic_capture_events -> llm_usage_events (event_pk) diesel::joinable!(agent_diagnostic_captures -> agent_sessions (session_pk)); diesel::joinable!(agent_diagnostic_captures -> app_users (user_id)); diesel::joinable!(agent_diagnostic_captures -> devices (device_context_id)); +diesel::joinable!(session_titles -> agent_sessions (session_pk)); diesel::allow_tables_to_appear_in_same_query!( agent_diagnostic_capture_events, @@ -178,5 +202,7 @@ diesel::allow_tables_to_appear_in_same_query!( devices, llm_usage_events, llm_usage_event_attachments, + schema_backfills, search_outbox, + session_titles, ); diff --git a/crates/abyss-backend/src/storage/postgres.rs b/crates/abyss-backend/src/storage/postgres.rs index 04caabc..2f995cf 100644 --- a/crates/abyss-backend/src/storage/postgres.rs +++ b/crates/abyss-backend/src/storage/postgres.rs @@ -3,6 +3,7 @@ mod repository; mod search; mod session_list; +mod session_titles; use std::time::Duration; @@ -46,6 +47,8 @@ impl PostgresEsBackend { db::run_migrations(&pool).map_err(|error| { AppError::internal(format!("run PostgreSQL migrations: {error}")) })?; + let mut connection = pool.get()?; + session_titles::backfill(&mut connection)?; } let search = config.search.as_ref().map(SearchService::new).transpose()?; diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index 8c1c052..f0439a4 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -55,6 +55,8 @@ use crate::{ }, }; +use super::session_titles; + #[cfg(test)] use crate::usage::persistence::{turn_identity_from_metadata, validate_event}; @@ -374,7 +376,10 @@ pub fn session_timeline( attachments_by_event.remove(&event_pk).unwrap_or_default(), )); } - Ok(assemble_timeline(session, turns, events_by_turn)) + let mut timeline = assemble_timeline(session, turns, events_by_turn); + (timeline.session.title, timeline.session.purpose) = + session_titles::load(connection, timeline.session.session_pk)?; + Ok(timeline) } type OutlineRow = ( @@ -478,7 +483,10 @@ pub fn session_outline( for event in events { events_by_turn.entry(event.turn_pk).or_default().push(event); } - Ok(assemble_timeline(session, turns, events_by_turn)) + let mut timeline = assemble_timeline(session, turns, events_by_turn); + (timeline.session.title, timeline.session.purpose) = + session_titles::load(connection, timeline.session.session_pk)?; + Ok(timeline) } fn load_session_turns( @@ -533,6 +541,8 @@ fn assemble_timeline( started_at: session.started_at, ended_at: session.ended_at, metadata: session.metadata, + title: None, + purpose: None, }, turns: turn_timelines, } @@ -699,6 +709,10 @@ fn ingest_one_event( )?; let tokens = normalized_tokens(event)?; let text = normalized_optional(event.text.as_deref()); + let naming_text = text + .as_deref() + .filter(|text| session_titles::is_candidate(event.event_type.as_str(), text)) + .map(str::to_owned); // `event_id` is the idempotency key supplied by the collector. Duplicate // observations are accepted as no-ops instead of failing the whole batch. @@ -737,23 +751,7 @@ fn ingest_one_event( // Child rows are written only for a newly inserted event. A replay must not // replace the attachments associated with the original idempotency key. if inserted == 1 && !attachments.is_empty() { - let attachment_rows = attachments - .into_iter() - .map(|attachment| NewUsageEventAttachment { - id: Uuid::now_v7(), - user_id, - event_pk: usage_event_id, - position: attachment.position, - media_type: attachment.media_type.as_str().to_owned(), - byte_size: attachment.byte_size, - sha256: attachment.sha256, - content: attachment.content, - created_at: now, - }) - .collect::>(); - diesel::insert_into(llm_usage_event_attachments::table) - .values(&attachment_rows) - .execute(connection)?; + insert_attachments(connection, user_id, usage_event_id, attachments, now)?; } // The enclosing transaction commits the event and its projection task @@ -766,11 +764,50 @@ fn ingest_one_event( created_at: now, }) .execute(connection)?; + session_titles::capture( + connection, + &session_titles::CapturedEvent { + user_id, + session_pk: session.id, + event_pk: usage_event_id, + turn_index, + event_type: event.event_type.as_str(), + text: naming_text.as_deref(), + observed_at, + }, + )?; } Ok(inserted == 1) } +fn insert_attachments( + connection: &mut PgConnection, + user_id: Uuid, + event_pk: Uuid, + attachments: Vec, + now: DateTime, +) -> Result<(), AppError> { + let attachment_rows = attachments + .into_iter() + .map(|attachment| NewUsageEventAttachment { + id: Uuid::now_v7(), + user_id, + event_pk, + position: attachment.position, + media_type: attachment.media_type.as_str().to_owned(), + byte_size: attachment.byte_size, + sha256: attachment.sha256, + content: attachment.content, + created_at: now, + }) + .collect::>(); + diesel::insert_into(llm_usage_event_attachments::table) + .values(&attachment_rows) + .execute(connection)?; + Ok(()) +} + fn upsert_device( connection: &mut PgConnection, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/postgres/session_list.rs b/crates/abyss-backend/src/storage/postgres/session_list.rs index d9616d7..e89ec06 100644 --- a/crates/abyss-backend/src/storage/postgres/session_list.rs +++ b/crates/abyss-backend/src/storage/postgres/session_list.rs @@ -31,9 +31,12 @@ const ACTIVITY_CTE: &str = "\ COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id)::bigint AS turns, \ COUNT(e.id)::bigint AS events, \ - COALESCE(SUM(e.total_tokens), 0)::bigint AS total_tokens \ + COALESCE(SUM(e.total_tokens), 0)::bigint AS total_tokens, \ + st.title AS title, \ + st.purpose AS purpose \ FROM agent_sessions s \ JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN session_titles st ON st.session_pk = s.id \ LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ WHERE s.user_id = $1 \ AND ($2 = false OR s.agent_name = ANY($3)) \ @@ -43,7 +46,7 @@ const ACTIVITY_CTE: &str = "\ AND ($10 = false OR EXISTS ( \ SELECT 1 FROM llm_usage_events m \ WHERE m.session_pk = s.id AND m.llm_model = $11)) \ - GROUP BY s.id, d.host_name, d.platform \ + GROUP BY s.id, d.host_name, d.platform, st.title, st.purpose \ ), \ filtered AS ( \ SELECT * FROM activity \ @@ -77,6 +80,10 @@ struct SessionListRow { events: i64, #[diesel(sql_type = BigInt)] total_tokens: i64, + #[diesel(sql_type = Nullable)] + title: Option, + #[diesel(sql_type = Nullable)] + purpose: Option, } #[derive(QueryableByName)] @@ -147,6 +154,8 @@ pub(super) fn list_sessions( turns: row.turns, events: row.events, total_tokens: row.total_tokens, + title: row.title, + purpose: row.purpose, }) .collect(); Ok(SessionListResponse::new(query, total, items)) diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs new file mode 100644 index 0000000..5db09d0 --- /dev/null +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -0,0 +1,239 @@ +//! PostgreSQL persistence of session names read from title-generation calls. + +use chrono::{DateTime, Utc}; +use diesel::{ + BoolExpressionMethods, Connection, ExpressionMethods, OptionalExtension, PgConnection, + QueryDsl, RunQueryDsl, TextExpressionMethods, +}; +use uuid::Uuid; + +use crate::{ + db::schema::{llm_usage_events, schema_backfills, session_titles}, + error::AppError, + usage::session_title::{ + TITLE_GENERATION_PURPOSE, TitleTask, may_carry_title, title_from_response, + }, +}; + +/// Marks the one-time scan of events stored before titles were captured. +const BACKFILL_NAME: &str = "session_titles_v1"; + +/// One stored event considered for title capture. +pub(super) struct CapturedEvent<'a> { + pub(super) user_id: Uuid, + pub(super) session_pk: Uuid, + pub(super) event_pk: Uuid, + pub(super) turn_index: i32, + pub(super) event_type: &'a str, + pub(super) text: Option<&'a str>, + pub(super) observed_at: DateTime, +} + +/// A naming call recognized from one of its events. +struct NamingCall { + task: TitleTask, + /// Title with the response event and observation time that carried it. + title: Option<(String, Uuid, DateTime)>, +} + +/// Returns `true` when an event's text could belong to a naming call, so +/// ingestion copies text only for candidates. +pub(super) fn is_candidate(event_type: &str, text: &str) -> bool { + match event_type { + "request" => TitleTask::from_request(text).is_some(), + "response" => may_carry_title(text), + _ => false, + } +} + +/// Records the session title when the event completes a naming call. +/// +/// A naming request and its response arrive as separate events in either +/// order, so each side looks up the other within the same turn. +pub(super) fn capture( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + naming_call(connection, event)?.map_or_else(|| Ok(()), |call| store(connection, event, call)) +} + +fn naming_call( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, +) -> Result, AppError> { + let Some(text) = event.text else { + return Ok(None); + }; + match event.event_type { + "request" => { + let Some(task) = TitleTask::from_request(text) else { + return Ok(None); + }; + let response = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(event.session_pk)) + .filter(llm_usage_events::turn_index.eq(event.turn_index)) + .filter(llm_usage_events::event_type.eq("response")) + .filter(llm_usage_events::observed_at.ge(event.observed_at)) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::text, + llm_usage_events::observed_at, + )) + .first::<(Uuid, Option, DateTime)>(connection) + .optional()?; + let title = response.and_then(|(id, text, observed_at)| { + text.as_deref() + .and_then(title_from_response) + .map(|title| (title, id, observed_at)) + }); + Ok(Some(NamingCall { task, title })) + } + "response" if may_carry_title(text) => { + let Some(title) = title_from_response(text) else { + return Ok(None); + }; + let request_text = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(event.session_pk)) + .filter(llm_usage_events::turn_index.eq(event.turn_index)) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::observed_at.le(event.observed_at)) + .order_by(llm_usage_events::observed_at.desc()) + .select(llm_usage_events::text) + .first::>(connection) + .optional()? + .flatten(); + Ok(request_text + .as_deref() + .and_then(TitleTask::from_request) + .map(|task| NamingCall { + task, + title: Some((title, event.event_pk, event.observed_at)), + })) + } + _ => Ok(None), + } +} + +fn store( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, + call: NamingCall, +) -> Result<(), AppError> { + let now = Utc::now(); + let purpose = + (call.task == TitleTask::AuxiliarySession).then(|| TITLE_GENERATION_PURPOSE.to_owned()); + let (title, title_event_pk, title_observed_at) = match call.title { + Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), + None => (None, None, None), + }; + let existing = session_titles::table + .find(event.session_pk) + .select((session_titles::title_observed_at, session_titles::purpose)) + .first::<(Option>, Option)>(connection) + .optional()?; + let Some((stored_at, stored_purpose)) = existing else { + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(event.session_pk), + session_titles::user_id.eq(event.user_id), + session_titles::title.eq(title), + session_titles::title_event_pk.eq(title_event_pk), + session_titles::title_observed_at.eq(title_observed_at), + session_titles::purpose.eq(purpose), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + return Ok(()); + }; + // A late or replayed naming call must not replace a newer title. + let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); + let target = session_titles::table.find(event.session_pk); + if newer { + diesel::update(target) + .set(( + session_titles::title.eq(title), + session_titles::title_event_pk.eq(title_event_pk), + session_titles::title_observed_at.eq(title_observed_at), + session_titles::purpose.eq(purpose.or(stored_purpose)), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + } else if stored_purpose.is_none() && purpose.is_some() { + diesel::update(target) + .set(( + session_titles::purpose.eq(purpose), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + } + Ok(()) +} + +/// Captures titles from events stored before capture existed, once. +pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { + connection.transaction(|transaction| { + let done = schema_backfills::table + .find(BACKFILL_NAME) + .select(schema_backfills::name) + .first::(transaction) + .optional()? + .is_some(); + if done { + return Ok(()); + } + let requests = + llm_usage_events::table + .filter(llm_usage_events::event_type.eq("request")) + .filter( + llm_usage_events::text + .like("%You are naming a coding session%") + .or(llm_usage_events::text + .like("%Generate a concise, single-line task title%")), + ) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::session_pk, + llm_usage_events::turn_index, + llm_usage_events::text, + llm_usage_events::observed_at, + )) + .load::<(Uuid, Uuid, Uuid, i32, Option, DateTime)>(transaction)?; + for (event_pk, user_id, session_pk, turn_index, text, observed_at) in requests { + capture( + transaction, + &CapturedEvent { + user_id, + session_pk, + event_pk, + turn_index, + event_type: "request", + text: text.as_deref(), + observed_at, + }, + )?; + } + diesel::insert_into(schema_backfills::table) + .values(( + schema_backfills::name.eq(BACKFILL_NAME), + schema_backfills::completed_at.eq(Utc::now()), + )) + .execute(transaction)?; + Ok::<_, AppError>(()) + }) +} + +/// Returns the stored title and purpose of one session. +pub(super) fn load( + connection: &mut PgConnection, + session_pk: Uuid, +) -> Result<(Option, Option), AppError> { + Ok(session_titles::table + .find(session_pk) + .select((session_titles::title, session_titles::purpose)) + .first::<(Option, Option)>(connection) + .optional()? + .unwrap_or_default()) +} diff --git a/crates/abyss-backend/src/storage/sqlite/mod.rs b/crates/abyss-backend/src/storage/sqlite/mod.rs index 9cf4669..aa39e70 100644 --- a/crates/abyss-backend/src/storage/sqlite/mod.rs +++ b/crates/abyss-backend/src/storage/sqlite/mod.rs @@ -7,6 +7,7 @@ mod repository; mod schema; mod search; mod session_list; +mod session_titles; use diesel::{QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star}; use uuid::Uuid; @@ -40,6 +41,7 @@ impl SqliteFtsBackend { connection::configure_database(&mut database)?; if config.run_migrations { migrations::run(&mut database)?; + session_titles::backfill(&mut database)?; } } Ok(Self { pool }) diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index f21bbc8..6600c5c 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -46,6 +46,7 @@ use super::{ agent_diagnostic_capture_events, agent_diagnostic_captures, agent_sessions, agent_turns, app_users, devices, llm_usage_event_attachments, llm_usage_events, }, + session_titles, }; pub(super) fn ingest_events( @@ -172,6 +173,30 @@ fn ingest_one_event( return Ok(false); } + insert_attachments(connection, user_id, event_pk, attachments, now)?; + insert_search_projection(connection, event_pk, user_id, session_pk, event)?; + session_titles::capture( + connection, + &session_titles::CapturedEvent { + user_id, + session_pk, + event_pk, + turn_index, + event_type: event.event_type.as_str(), + text: text.as_deref(), + observed_at, + }, + )?; + Ok(true) +} + +fn insert_attachments( + connection: &mut SqliteConnection, + user_id: Uuid, + event_pk: Uuid, + attachments: Vec, + now: i64, +) -> Result<(), AppError> { let attachment_rows = attachments .into_iter() .map(|attachment| NewAttachment { @@ -192,8 +217,7 @@ fn ingest_one_event( .execute(connection)?; } - insert_search_projection(connection, event_pk, user_id, session_pk, event)?; - Ok(true) + Ok(()) } fn upsert_device( @@ -649,7 +673,10 @@ pub(super) fn session_timeline( attachments.remove(&event_pk).unwrap_or_default(), )); } - Ok(assemble_timeline(session, turns, events_by_turn)) + let mut timeline = assemble_timeline(session, turns, events_by_turn); + (timeline.session.title, timeline.session.purpose) = + session_titles::load(connection, timeline.session.session_pk)?; + Ok(timeline) } type OutlineRow = ( @@ -778,7 +805,10 @@ pub(super) fn session_outline( for event in events { events_by_turn.entry(event.turn_pk).or_default().push(event); } - Ok(assemble_timeline(session, turns, events_by_turn)) + let mut timeline = assemble_timeline(session, turns, events_by_turn); + (timeline.session.title, timeline.session.purpose) = + session_titles::load(connection, timeline.session.session_pk)?; + Ok(timeline) } /// Computes the outline length and preview in Rust for text SQLite cannot measure. @@ -848,6 +878,8 @@ fn assemble_timeline( started_at: session.started_at, ended_at: session.ended_at, metadata: session.metadata, + title: None, + purpose: None, }, turns: turn_timelines, } diff --git a/crates/abyss-backend/src/storage/sqlite/schema.rs b/crates/abyss-backend/src/storage/sqlite/schema.rs index 80531fe..af9182f 100644 --- a/crates/abyss-backend/src/storage/sqlite/schema.rs +++ b/crates/abyss-backend/src/storage/sqlite/schema.rs @@ -133,6 +133,29 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::{BigInt, Nullable, Text}; + + session_titles (session_pk) { + session_pk -> Text, + user_id -> Text, + title -> Nullable, + title_event_pk -> Nullable, + title_observed_at -> Nullable, + purpose -> Nullable, + updated_at -> BigInt, + } +} + +diesel::table! { + use diesel::sql_types::{BigInt, Text}; + + schema_backfills (name) { + name -> Text, + completed_at -> BigInt, + } +} + diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -149,6 +172,7 @@ diesel::joinable!(agent_diagnostic_capture_events -> llm_usage_events (event_pk) diesel::joinable!(agent_diagnostic_captures -> agent_sessions (session_pk)); diesel::joinable!(agent_diagnostic_captures -> app_users (user_id)); diesel::joinable!(agent_diagnostic_captures -> devices (device_context_id)); +diesel::joinable!(session_titles -> agent_sessions (session_pk)); diesel::allow_tables_to_appear_in_same_query!( agent_diagnostic_capture_events, @@ -159,4 +183,6 @@ diesel::allow_tables_to_appear_in_same_query!( devices, llm_usage_events, llm_usage_event_attachments, + schema_backfills, + session_titles, ); diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index 4481cb3..095f5db 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -35,9 +35,12 @@ const ACTIVITY_CTE: &str = "\ COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id) AS turns, \ COUNT(e.id) AS events, \ - COALESCE(SUM(e.total_tokens), 0) AS total_tokens \ + COALESCE(SUM(e.total_tokens), 0) AS total_tokens, \ + st.title AS title, \ + st.purpose AS purpose \ FROM agent_sessions s \ JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN session_titles st ON st.session_pk = s.id \ LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ WHERE s.user_id = ?1 \ AND (?2 = 0 OR s.agent_name IN (SELECT value FROM json_each(?3))) \ @@ -81,6 +84,10 @@ struct SessionListRow { events: i64, #[diesel(sql_type = BigInt)] total_tokens: i64, + #[diesel(sql_type = Nullable)] + title: Option, + #[diesel(sql_type = Nullable)] + purpose: Option, } #[derive(QueryableByName)] @@ -154,6 +161,8 @@ pub(super) fn list_sessions( turns: row.turns, events: row.events, total_tokens: row.total_tokens, + title: row.title, + purpose: row.purpose, }) }) .collect::, AppError>>()?; diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs new file mode 100644 index 0000000..9939fb5 --- /dev/null +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -0,0 +1,488 @@ +//! SQLite persistence of session names read from title-generation calls. + +use chrono::Utc; +use diesel::{ + BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, + SqliteConnection, TextExpressionMethods, +}; +use uuid::Uuid; + +use super::{ + models::parse_uuid, + schema::{llm_usage_events, schema_backfills, session_titles}, +}; +use crate::{ + error::AppError, + usage::session_title::{ + TITLE_GENERATION_PURPOSE, TitleTask, may_carry_title, title_from_response, + }, +}; + +/// Marks the one-time scan of events stored before titles were captured. +const BACKFILL_NAME: &str = "session_titles_v1"; + +/// One stored event considered for title capture. +pub(super) struct CapturedEvent<'a> { + pub(super) user_id: Uuid, + pub(super) session_pk: Uuid, + pub(super) event_pk: Uuid, + pub(super) turn_index: i32, + pub(super) event_type: &'a str, + pub(super) text: Option<&'a str>, + pub(super) observed_at: i64, +} + +/// A naming call recognized from one of its events. +struct NamingCall { + task: TitleTask, + /// Title with the response event and observation time that carried it. + title: Option<(String, String, i64)>, +} + +/// Records the session title when the event completes a naming call. +/// +/// A naming request and its response arrive as separate events in either +/// order, so each side looks up the other within the same turn. +pub(super) fn capture( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + naming_call(connection, event)?.map_or_else(|| Ok(()), |call| store(connection, event, call)) +} + +fn naming_call( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, +) -> Result, AppError> { + let Some(text) = event.text else { + return Ok(None); + }; + let session = event.session_pk.to_string(); + match event.event_type { + "request" => { + let Some(task) = TitleTask::from_request(text) else { + return Ok(None); + }; + let response = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(&session)) + .filter(llm_usage_events::turn_index.eq(event.turn_index)) + .filter(llm_usage_events::event_type.eq("response")) + .filter(llm_usage_events::observed_at.ge(event.observed_at)) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::text, + llm_usage_events::observed_at, + )) + .first::<(String, Option, i64)>(connection) + .optional()?; + let title = response.and_then(|(id, text, observed_at)| { + text.as_deref() + .and_then(title_from_response) + .map(|title| (title, id, observed_at)) + }); + Ok(Some(NamingCall { task, title })) + } + "response" if may_carry_title(text) => { + let Some(title) = title_from_response(text) else { + return Ok(None); + }; + let request_text = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(&session)) + .filter(llm_usage_events::turn_index.eq(event.turn_index)) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::observed_at.le(event.observed_at)) + .order_by(llm_usage_events::observed_at.desc()) + .select(llm_usage_events::text) + .first::>(connection) + .optional()? + .flatten(); + Ok(request_text + .as_deref() + .and_then(TitleTask::from_request) + .map(|task| NamingCall { + task, + title: Some((title, event.event_pk.to_string(), event.observed_at)), + })) + } + _ => Ok(None), + } +} + +fn store( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, + call: NamingCall, +) -> Result<(), AppError> { + let session = event.session_pk.to_string(); + let now = Utc::now().timestamp_micros(); + let purpose = + (call.task == TitleTask::AuxiliarySession).then(|| TITLE_GENERATION_PURPOSE.to_owned()); + let (title, title_event_pk, title_observed_at) = match call.title { + Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), + None => (None, None, None), + }; + let existing = session_titles::table + .find(&session) + .select((session_titles::title_observed_at, session_titles::purpose)) + .first::<(Option, Option)>(connection) + .optional()?; + let Some((stored_at, stored_purpose)) = existing else { + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(&session), + session_titles::user_id.eq(event.user_id.to_string()), + session_titles::title.eq(title), + session_titles::title_event_pk.eq(title_event_pk), + session_titles::title_observed_at.eq(title_observed_at), + session_titles::purpose.eq(purpose), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + return Ok(()); + }; + // A late or replayed naming call must not replace a newer title. + let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); + let target = session_titles::table.find(&session); + if newer { + diesel::update(target) + .set(( + session_titles::title.eq(title), + session_titles::title_event_pk.eq(title_event_pk), + session_titles::title_observed_at.eq(title_observed_at), + session_titles::purpose.eq(purpose.or(stored_purpose)), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + } else if stored_purpose.is_none() && purpose.is_some() { + diesel::update(target) + .set(( + session_titles::purpose.eq(purpose), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + } + Ok(()) +} + +/// Captures titles from events stored before capture existed, once. +pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { + let done = schema_backfills::table + .find(BACKFILL_NAME) + .select(schema_backfills::name) + .first::(connection) + .optional()? + .is_some(); + if done { + return Ok(()); + } + connection.immediate_transaction(|transaction| { + let requests = + llm_usage_events::table + .filter(llm_usage_events::event_type.eq("request")) + .filter( + llm_usage_events::text + .like("%You are naming a coding session%") + .or(llm_usage_events::text + .like("%Generate a concise, single-line task title%")), + ) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::session_pk, + llm_usage_events::turn_index, + llm_usage_events::text, + llm_usage_events::observed_at, + )) + .load::<(String, String, String, i32, Option, i64)>(transaction)?; + for (event_pk, user_id, session_pk, turn_index, text, observed_at) in requests { + capture( + transaction, + &CapturedEvent { + user_id: parse_uuid(&user_id, "event user id")?, + session_pk: parse_uuid(&session_pk, "event session id")?, + event_pk: parse_uuid(&event_pk, "event id")?, + turn_index, + event_type: "request", + text: text.as_deref(), + observed_at, + }, + )?; + } + diesel::insert_into(schema_backfills::table) + .values(( + schema_backfills::name.eq(BACKFILL_NAME), + schema_backfills::completed_at.eq(Utc::now().timestamp_micros()), + )) + .execute(transaction)?; + Ok::<_, AppError>(()) + }) +} + +/// Returns the stored title and purpose of one session. +pub(super) fn load( + connection: &mut SqliteConnection, + session_pk: Uuid, +) -> Result<(Option, Option), AppError> { + Ok(session_titles::table + .find(session_pk.to_string()) + .select((session_titles::title, session_titles::purpose)) + .first::<(Option, Option)>(connection) + .optional()? + .unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use super::{ + super::{ + migrations, + repository::ingest_events, + schema::{agent_sessions, schema_backfills, session_titles}, + }, + backfill, load, + }; + use crate::usage::IngestEventsRequest; + + const OWNER_ID: Uuid = Uuid::from_u128(1); + const CLAUDE_NAMING: &str = "You are naming a coding session so the user can pick it out"; + const CODEX_NAMING: &str = "Generate a concise, single-line task title of at most 36"; + + fn database() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + database + } + + fn event(agent: &str, session: &str, id: &str, at: &str, kind: &str, text: &str) -> Value { + json!({ + "event_id": id, + "observed_at": at, + "device": {"host_name": "local", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": session, + "turn_index": 2, + "llm": {"provider": "anthropic", "model": "claude-haiku"}, + "event_type": kind, + "text": text, + "token_usage": {"output_tokens": 1}, + "metadata": {} + }) + } + + fn ingest(database: &mut SqliteConnection, events: &[Value]) { + let request: IngestEventsRequest = + serde_json::from_value(json!({ "events": events })).expect("request should parse"); + ingest_events(database, &request, OWNER_ID, 100).expect("ingest should succeed"); + } + + fn session_title( + database: &mut SqliteConnection, + session: &str, + ) -> (Option, Option) { + let pk = agent_sessions::table + .filter(agent_sessions::session_id.eq(session)) + .select(agent_sessions::id) + .first::(database) + .expect("session should exist"); + load(database, Uuid::parse_str(&pk).expect("uuid")).expect("title should load") + } + + #[test] + fn names_a_claude_code_session_from_its_own_naming_call() { + let mut database = database(); + ingest( + &mut database, + &[ + event( + "claude-code", + "cc", + "r1", + "2026-09-30T00:00:00Z", + "request", + "fix the build", + ), + event( + "claude-code", + "cc", + "s1", + "2026-09-30T00:00:01Z", + "response", + "Done.", + ), + event( + "claude-code", + "cc", + "r2", + "2026-09-30T00:00:02Z", + "request", + CLAUDE_NAMING, + ), + event( + "claude-code", + "cc", + "s2", + "2026-09-30T00:00:03Z", + "response", + r#"{"title":"Build fix"}"#, + ), + ], + ); + + assert_eq!( + session_title(&mut database, "cc"), + (Some("Build fix".to_owned()), None) + ); + } + + #[test] + fn marks_codex_naming_threads_as_auxiliary_even_when_the_response_arrives_first() { + let mut database = database(); + ingest( + &mut database, + &[event( + "codex", + "title-thread", + "s1", + "2026-09-30T00:00:01Z", + "response", + "{\"title\":\"Draw a pelican\"}\n\n{\"title\":\"Draw a pelican\"}", + )], + ); + ingest( + &mut database, + &[event( + "codex", + "title-thread", + "r1", + "2026-09-30T00:00:00Z", + "request", + CODEX_NAMING, + )], + ); + + assert_eq!( + session_title(&mut database, "title-thread"), + ( + Some("Draw a pelican".to_owned()), + Some("title_generation".to_owned()) + ) + ); + } + + #[test] + fn keeps_the_newest_title_and_ignores_quoted_instructions() { + let mut database = database(); + ingest( + &mut database, + &[ + event( + "claude-code", + "cc", + "r2", + "2026-09-30T02:00:00Z", + "request", + CLAUDE_NAMING, + ), + event( + "claude-code", + "cc", + "s2", + "2026-09-30T02:00:01Z", + "response", + r#"{"title":"Newer"}"#, + ), + event( + "claude-code", + "cc", + "r1", + "2026-09-30T01:00:00Z", + "request", + CLAUDE_NAMING, + ), + event( + "claude-code", + "cc", + "s1", + "2026-09-30T01:00:01Z", + "response", + r#"{"title":"Older"}"#, + ), + event( + "claude-code", + "cc", + "r3", + "2026-09-30T03:00:00Z", + "request", + CLAUDE_NAMING, + ), + event( + "claude-code", + "cc", + "s3", + "2026-09-30T03:00:01Z", + "response", + "That prompt names sessions.", + ), + ], + ); + + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Newer") + ); + } + + #[test] + fn backfills_stored_naming_calls_once() { + let mut database = database(); + ingest( + &mut database, + &[ + event( + "claude-code", + "cc", + "r1", + "2026-09-30T00:00:00Z", + "request", + CLAUDE_NAMING, + ), + event( + "claude-code", + "cc", + "s1", + "2026-09-30T00:00:01Z", + "response", + r#"{"title":"Stored"}"#, + ), + ], + ); + diesel::delete(session_titles::table) + .execute(&mut database) + .expect("clear titles"); + + backfill(&mut database).expect("backfill should succeed"); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Stored") + ); + + diesel::delete(session_titles::table) + .execute(&mut database) + .expect("clear titles"); + backfill(&mut database).expect("second backfill should succeed"); + assert_eq!(session_title(&mut database, "cc"), (None, None)); + assert_eq!( + schema_backfills::table + .count() + .get_result::(&mut database) + .expect("count"), + 1 + ); + } +} diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index a2b1cd7..ddec30d 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -13,6 +13,7 @@ pub mod event_order; pub mod persistence; /// Owner-scoped session index contracts. pub mod session_list; +pub mod session_title; pub mod trajectory; use chrono::{DateTime, Utc}; @@ -529,6 +530,11 @@ pub struct SessionInfo { pub ended_at: Option>, /// Stable, allowlisted session metadata. pub metadata: Value, + /// Name the Agent generated for the session, when one was captured. + pub title: Option, + /// Why the session exists when it only serves the Agent, such as + /// `title_generation`; `None` for ordinary sessions. + pub purpose: Option, } /// Extracts the endpoint working directory retained in session metadata. diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index dff6cb1..192b593 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -110,6 +110,11 @@ pub struct SessionListItem { pub turns: i64, pub events: i64, pub total_tokens: i64, + /// Name the Agent generated for the session, when one was captured. + pub title: Option, + /// Why the session exists when it only serves the Agent, such as + /// `title_generation`; `None` for ordinary sessions. + pub purpose: Option, } impl SessionListResponse { diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs new file mode 100644 index 0000000..55d1b8c --- /dev/null +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -0,0 +1,130 @@ +//! Session names recovered from the Agents' own title-generation calls. +//! +//! Claude Code and Codex ask a model to name each session and record the +//! answer as `{"title": "..."}`. Those calls are captured like any other, so +//! the name can be read back from them without guessing: a call counts only +//! when its request carries an official naming instruction and its response +//! consists solely of title objects that agree. + +use serde::Deserialize; + +/// Claude Code instruction for naming the session it runs in. +const CLAUDE_CODE_NAMING_INSTRUCTION: &str = "You are naming a coding session"; +/// Codex instruction for naming a task; Codex sends it from its own thread. +const CODEX_NAMING_INSTRUCTION: &str = "Generate a concise, single-line task title"; +/// Longest title kept, in characters; longer answers are not titles. +const MAX_TITLE_CHARS: usize = 200; + +/// Session purpose stored for Codex's standalone naming threads. +pub const TITLE_GENERATION_PURPOSE: &str = "title_generation"; + +/// Where a naming call's title belongs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TitleTask { + /// The call names the session it was sent from (Claude Code). + NamesOwnSession, + /// The call runs in a separate naming session (Codex); that session is + /// auxiliary and the title names the naming session itself. + AuxiliarySession, +} + +impl TitleTask { + /// Recognizes a naming call from its request text. + pub fn from_request(text: &str) -> Option { + if text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) { + Some(Self::NamesOwnSession) + } else if text.contains(CODEX_NAMING_INSTRUCTION) { + Some(Self::AuxiliarySession) + } else { + None + } + } +} + +#[derive(Deserialize)] +struct TitleObject { + title: String, +} + +/// Reads the title from a naming response. +/// +/// Streaming clients may repeat the answer, so several identical objects are +/// accepted. Anything else, such as reasoning text or a normal reply to a +/// user who quoted the naming instruction, yields no title. +pub fn title_from_response(text: &str) -> Option { + let mut title: Option = None; + for value in serde_json::Deserializer::from_str(text.trim()).into_iter::() { + let candidate = value.ok()?.title.trim().to_owned(); + if candidate.is_empty() || candidate.chars().count() > MAX_TITLE_CHARS { + return None; + } + match &title { + Some(existing) if *existing != candidate => return None, + Some(_) => {} + None => title = Some(candidate), + } + } + title +} + +/// Returns `true` when a response text could carry a title, so ingestion can +/// skip the pairing lookup for ordinary replies. +pub fn may_carry_title(text: &str) -> bool { + let trimmed = text.trim_start(); + trimmed.starts_with('{') && trimmed.contains("\"title\"") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognizes_official_naming_instructions() { + assert_eq!( + TitleTask::from_request("system: …\n\nYou are naming a coding session so the user…"), + Some(TitleTask::NamesOwnSession) + ); + assert_eq!( + TitleTask::from_request("Generate a concise, single-line task title of at most 36"), + Some(TitleTask::AuxiliarySession) + ); + assert_eq!(TitleTask::from_request("please name this session"), None); + } + + #[test] + fn reads_single_and_repeated_title_objects() { + assert_eq!( + title_from_response(r#"{"title":"Timeline颗粒度细化"}"#).as_deref(), + Some("Timeline颗粒度细化") + ); + assert_eq!( + title_from_response("{\"title\":\"Send a ping\"}\n\n{\"title\":\"Send a ping\"}") + .as_deref(), + Some("Send a ping") + ); + } + + #[test] + fn rejects_everything_that_is_not_one_agreed_title() { + for text in [ + "We need respond title only, imperative verb, under 5 words", + r#"{"title":"A"}{"title":"B"}"#, + r#"{"title":""}"#, + r#"{"name":"x"}"#, + r#"Here is the title: {"title":"x"}"#, + "", + ] { + assert_eq!(title_from_response(text), None, "{text:?}"); + } + assert_eq!( + title_from_response(&format!(r#"{{"title":"{}"}}"#, "x".repeat(201))), + None + ); + } + + #[test] + fn prefilters_candidate_responses() { + assert!(may_carry_title(" {\"title\": \"x\"}")); + assert!(!may_carry_title("The title is x")); + } +} diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 18efc94..5d7f11b 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -378,6 +378,8 @@ mod tests { started_at: now, ended_at: None, metadata: json!({}), + title: None, + purpose: None, }, turns: vec![TurnTimeline { turn_pk: Uuid::nil(), From 9904c1299aa8f8939a831fa3ebceec4f90f8d1c8 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 04:56:07 -0700 Subject: [PATCH 06/30] fix(trajectory): deduplicate replayed snapshots by message identity and order --- crates/abyss-backend/src/usage/trajectory.rs | 185 +++++++++++++++++- .../src/usage/trajectory/conversation.rs | 147 ++++++++++++++ 2 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 crates/abyss-backend/src/usage/trajectory/conversation.rs diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 5d7f11b..0017b79 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -5,6 +5,10 @@ //! new request paragraphs, first-seen tool results, and response text and tool //! calls, with each tool call paired to its result by call identifier. +mod conversation; + +use conversation::ConversationMerger; + use std::{ collections::{HashMap, HashSet, hash_map::DefaultHasher}, hash::{Hash, Hasher}, @@ -128,6 +132,7 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { #[derive(Default)] struct Merger { + conversation: ConversationMerger, items: Vec, seen_paragraphs: HashSet, seen_results: HashSet, @@ -141,6 +146,21 @@ struct Merger { impl Merger { fn request(&mut self, event: &UsageEventResponse) { + if let Some(items) = self.conversation.project(event, self.current_stream) { + self.items.extend(items); + } else { + self.legacy_request_text(event); + } + let stream = self.current_stream; + for segment in content_segments(&event.metadata) { + if segment_str(segment, "type") == Some("tool_call") { + self.tool_call(event, segment, stream); + } + } + self.request_tools(event, stream); + } + + fn legacy_request_text(&mut self, event: &UsageEventResponse) { let text = event.text.as_deref().unwrap_or_default(); let paragraphs = text.split(PARAGRAPH_SEPARATOR).collect::>(); let first = paragraphs.first().copied().unwrap_or_default(); @@ -189,7 +209,9 @@ impl Merger { } self.last_paragraphs .insert(stream, paragraphs.into_iter().map(str::to_owned).collect()); + } + fn request_tools(&mut self, event: &UsageEventResponse, stream: usize) { for segment in content_segments(&event.metadata) { if segment_str(segment, "type") != Some("tool_result") { continue; @@ -221,7 +243,9 @@ impl Merger { fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; - if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { + if let Some(items) = self.conversation.project(event, stream) { + self.items.extend(items); + } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); item.text = Some(text.to_owned()); self.items.push(item); @@ -230,16 +254,23 @@ impl Merger { if segment_str(segment, "type") != Some("tool_call") { continue; } - let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); - item.name = segment_str(segment, "name").map(str::to_owned); - item.call_id = segment_str(segment, "call_id").map(str::to_owned); - item.input = segment_str(segment, "input").map(str::to_owned); - if let Some(call_id) = &item.call_id { - self.calls.insert(call_id.clone(), self.items.len()); - } - self.items.push(item); + self.tool_call(event, segment, stream); } } + + fn tool_call(&mut self, event: &UsageEventResponse, segment: &Value, stream: usize) { + if segment_str(segment, "call_id").is_some_and(|id| self.calls.contains_key(id)) { + return; + } + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); + item.name = segment_str(segment, "name").map(str::to_owned); + item.call_id = segment_str(segment, "call_id").map(str::to_owned); + item.input = segment_str(segment, "input").map(str::to_owned); + if let Some(call_id) = &item.call_id { + self.calls.insert(call_id.clone(), self.items.len()); + } + self.items.push(item); + } } /// Identity of a request paragraph. The system fragment's first line carries @@ -360,6 +391,142 @@ mod tests { assert!(items[2].continued); } + #[test] + fn snapshots_project_replay_once_and_keep_repeated_user_occurrences() { + let system = json!({"role": "system", "text": "system instructions"}); + let developer = json!({"role": "developer", "text": "developer instructions"}); + let user = json!({"role": "user", "text": "继续"}); + let assistant = json!({"id": "a1", "role": "assistant", "text": "working"}); + let final_reply = json!({"id": "a2", "role": "assistant", "text": "done"}); + let first = vec![system, developer, user.clone()]; + let mut replay = first.clone(); + replay.push(assistant.clone()); + let call = json!({"type": "tool_call", "call_id": "c1", "name": "exec", "input": "run"}); + let mut response = snapshot_event("response", &[assistant], "r1", None); + response.metadata["content_segments"] = json!([call]); + let mut continuation = snapshot_event("request", &replay, "r2", None); + continuation.metadata["content_segments"] = json!([ + call, {"type": "tool_result", "call_id": "c1", "output": "success"} + ]); + replay.extend([final_reply.clone(), user]); + let items = build(timeline(vec![ + snapshot_event("request", &first, "r1", None), + response, + continuation, + snapshot_event("response", &[final_reply], "r2", None), + snapshot_event("request", &replay, "r3", None), + ])) + .items; + + assert_eq!( + items.iter().map(|item| item.kind).collect::>(), + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::Context, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::ToolCall, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User, + ] + ); + assert_eq!(items[2].text.as_deref(), Some("继续")); + assert_eq!(items[6].text.as_deref(), Some("继续")); + assert_eq!(items[4].output.as_deref(), Some("success")); + } + + #[test] + fn incremental_requests_append_identical_input_and_ignore_retries() { + let user = json!({"role": "user", "text": "继续"}); + let assistant = json!({"id": "a1", "role": "assistant", "text": "done"}); + let items = build(timeline(vec![ + snapshot_event("request", std::slice::from_ref(&user), "r1", None), + snapshot_event("response", std::slice::from_ref(&assistant), "r1", None), + snapshot_event("request", &[assistant, user.clone()], "r2", Some("r1")), + snapshot_event("request", &[user], "r2", Some("r1")), + ])) + .items; + assert_eq!(items.len(), 3); + assert_eq!(items[0].kind, TrajectoryItemKind::User); + assert_eq!(items[1].kind, TrajectoryItemKind::Assistant); + assert_eq!(items[2].kind, TrajectoryItemKind::User); + } + + #[test] + fn snapshots_use_roles_and_occurrences_instead_of_paragraphs_or_tags() { + let text = "system: quoted\n\nquoted"; + let user = json!({"role": "user", "text": text}); + let items = build(timeline(vec![ + snapshot_event("request", &[user.clone(), user.clone()], "r1", None), + snapshot_event("request", &[user.clone(), user], "r2", None), + ])) + .items; + assert_eq!(items.len(), 2); + assert!( + items + .iter() + .all(|item| item.kind == TrajectoryItemKind::User) + ); + assert!(items.iter().all(|item| item.text.as_deref() == Some(text))); + } + + #[test] + fn new_context_does_not_replay_existing_dialogue() { + let user = json!({"role": "user", "text": "hello"}); + let assistant = json!({"id": "a1", "role": "assistant", "text": "hi"}); + let items = build(timeline(vec![ + snapshot_event("request", std::slice::from_ref(&user), "r1", None), + snapshot_event("response", std::slice::from_ref(&assistant), "r1", None), + snapshot_event( + "request", + &[ + json!({"role": "developer", "text": "updated instructions"}), + user, + assistant, + ], + "r2", + None, + ), + ])) + .items; + assert_eq!(items.len(), 3); + assert_eq!(items[2].kind, TrajectoryItemKind::Context); + assert_eq!(items[2].text.as_deref(), Some("updated instructions")); + } + + #[test] + fn observed_item_ids_survive_compaction_and_distinguish_new_messages() { + let first = json!({"id": "u1", "role": "user", "text": "continue"}); + let second = json!({"id": "u2", "role": "user", "text": "continue"}); + let items = build(timeline(vec![ + snapshot_event("request", std::slice::from_ref(&first), "r1", None), + snapshot_event("request", &[second], "r2", None), + snapshot_event("request", &[first], "r3", None), + ])) + .items; + assert_eq!(items.len(), 2); + } + + fn snapshot_event( + side: &str, + messages: &[Value], + response: &str, + previous: Option<&str>, + ) -> UsageEventResponse { + let text = messages + .iter() + .filter_map(|message| message["text"].as_str()) + .collect::>() + .join("\n\n"); + event( + side, + &text, + json!({"conversation": { + "messages": messages, "response_id": response, "previous_response_id": previous, + }}), + ) + } + #[test] fn common_prefix_stops_on_character_boundaries() { assert_eq!(common_prefix_len("中文a", "中字b"), 3); diff --git a/crates/abyss-backend/src/usage/trajectory/conversation.rs b/crates/abyss-backend/src/usage/trajectory/conversation.rs new file mode 100644 index 0000000..a79d90a --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory/conversation.rs @@ -0,0 +1,147 @@ +//! Projects role-aware snapshots without changing stored request or response text. +//! +//! Match occurrences in order, rather than deduplicating strings globally: a +//! second identical user message is still a new message. Provider item ids are +//! used when available, and incremental requests append their new user input. + +use std::collections::{HashMap, HashSet}; + +use serde::Deserialize; + +use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; + +#[derive(Clone, Deserialize)] +struct Message { + #[serde(default)] + id: Option, + role: Role, + text: String, +} + +#[derive(Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Role { + System, + Developer, + User, + Assistant, + Reasoning, +} + +impl Role { + const fn kind(self) -> TrajectoryItemKind { + match self { + Self::System | Self::Developer => TrajectoryItemKind::Context, + Self::User => TrajectoryItemKind::User, + Self::Assistant | Self::Reasoning => TrajectoryItemKind::Assistant, + } + } + + const fn is_context(self) -> bool { + matches!(self, Self::System | Self::Developer) + } +} + +impl Message { + fn matches(&self, other: &Self) -> bool { + self.role == other.role + && self.text == other.text + && match (&self.id, &other.id) { + (Some(left), Some(right)) => left == right, + _ => true, + } + } +} + +#[derive(Deserialize)] +struct Snapshot { + messages: Vec, + response_id: Option, + previous_response_id: Option, +} + +#[derive(Default)] +pub(super) struct ConversationMerger { + history: Vec, + contexts: Vec, + identified: HashMap, + /// The same provider response may be observed again after a retry. + requests: HashSet, + responses: HashSet, +} + +impl ConversationMerger { + /// None keeps the existing legacy projection for events without snapshots. + pub(super) fn project( + &mut self, + event: &UsageEventResponse, + stream: usize, + ) -> Option> { + let snapshot: Snapshot = + serde_json::from_value(event.metadata.get("conversation")?.clone()).ok()?; + let is_request = event.event_type == "request"; + if let Some(id) = snapshot.response_id { + let seen = if is_request { + &mut self.requests + } else { + &mut self.responses + }; + if !seen.insert(id) { + return Some(Vec::new()); + } + } + let incremental = snapshot.previous_response_id.is_some(); + let mut cursor = 0; + let mut history = Vec::new(); + let mut added = Vec::new(); + for message in snapshot.messages { + let known_id = message + .id + .as_ref() + .and_then(|id| self.identified.get(id)) + .is_some_and(|earlier| earlier.matches(&message)); + let already_seen = if message.role.is_context() { + let known = self + .contexts + .iter() + .any(|earlier| earlier.matches(&message)); + if !known { + self.contexts.push(message.clone()); + } + known + } else if is_request && !incremental { + let found = self.history[cursor..] + .iter() + .position(|earlier| earlier.matches(&message)); + if let Some(offset) = found { + cursor = cursor.saturating_add(offset).saturating_add(1); + } + history.push(message.clone()); + known_id || found.is_some() + } else { + // Incremental user messages represent new occurrences even if + // they say the same thing as an earlier message. Replayed + // assistant items can still refer to an observed response. + let replay = is_request + && message.role != Role::User + && self.history.iter().any(|earlier| earlier.matches(&message)); + if !known_id && !replay { + self.history.push(message.clone()); + } + known_id || replay + }; + if let Some(id) = &message.id { + self.identified.insert(id.clone(), message.clone()); + } + if !already_seen && !message.text.trim().is_empty() { + let mut item = TrajectoryItem::new(message.role.kind(), stream, event); + item.text = Some(message.text); + added.push(item); + } + } + if is_request && !incremental { + self.history = history; + } + Some(added) + } +} From 45d1a656f787d515f7d96646904eb44ab8c9358e Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 06:16:13 -0700 Subject: [PATCH 07/30] feat(trajectory): show Codex context compaction as one item Codex compacts its context by sending a normal request that ends with a fixed "CONTEXT CHECKPOINT COMPACTION" prompt, and replays the answer at the start of the compacted history behind a fixed "Another language model started to solve this problem" sentence. The trajectory showed the prompt as user input, the summary as an assistant reply, and the summary again as user input. Recognize both fixed texts in the legacy and the snapshot projection. The prompt and the summary from its response form one compaction item, and the replayed handoff is hidden. When the summary request was not observed, the handoff alone records the compaction. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/usage/trajectory.rs | 231 ++++++++++++++++++- 1 file changed, 230 insertions(+), 1 deletion(-) diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 0017b79..77193e3 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -27,6 +27,13 @@ const PARAGRAPH_SEPARATOR: &str = "\n\n"; const SYSTEM_PREFIX: &str = "system: "; /// Shared prefix length above which a changed paragraph counts as grown. const GROWN_PARAGRAPH_MIN_PREFIX_BYTES: usize = 200; +/// Codex opens the request that asks the model for a context handoff summary +/// with this prompt when it compacts its context. +const COMPACTION_PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION."; +/// Codex opens the compacted history it sends afterwards with this sentence, +/// followed by the handoff summary. +const COMPACTION_HANDOFF: &str = "Another language model started to solve this problem and \ + produced a summary of its thinking process."; #[derive(Debug, Serialize)] /// Session information and its merged, de-duplicated trajectory items. @@ -51,6 +58,9 @@ pub enum TrajectoryItemKind { ToolCall, /// Request tool result whose call was not observed. ToolResult, + /// Context compaction: the summary prompt as `input` and the handoff + /// summary that replaced the earlier history as `output`. + Compaction, } #[derive(Debug, Serialize)] @@ -110,6 +120,25 @@ impl TrajectoryItem { output_event_pk: None, } } + + /// An empty item of another kind contributed by the same event. + fn sibling(&self, kind: TrajectoryItemKind) -> Self { + Self { + kind, + stream: self.stream, + turn_index: self.turn_index, + event_pk: self.event_pk, + observed_at: self.observed_at, + llm_model: self.llm_model.clone(), + text: None, + continued: false, + name: None, + call_id: None, + input: None, + output: None, + output_event_pk: None, + } + } } /// Merges an ordered session timeline into its trajectory. @@ -142,15 +171,22 @@ struct Merger { calls: HashMap, /// Stream of the most recent request; responses belong to it. current_stream: usize, + /// Compaction item whose summary the next response carries. + pending_compaction: Option, + /// Handoff summaries already shown, so their replay is not shown again. + summaries: Vec, } impl Merger { fn request(&mut self, event: &UsageEventResponse) { + self.pending_compaction = None; + let first_new = self.items.len(); if let Some(items) = self.conversation.project(event, self.current_stream) { self.items.extend(items); } else { self.legacy_request_text(event); } + self.mark_compaction(first_new); let stream = self.current_stream; for segment in content_segments(&event.metadata) { if segment_str(segment, "type") == Some("tool_call") { @@ -241,9 +277,79 @@ impl Merger { } } + /// Turns the compaction prompt into a compaction item and hides the + /// handoff summary when the compacted history replays it. + fn mark_compaction(&mut self, first_new: usize) { + for mut item in self.items.split_off(first_new) { + let Some(text) = item + .text + .clone() + .filter(|_| item.kind == TrajectoryItemKind::User) + else { + self.items.push(item); + continue; + }; + // Legacy projections join all new paragraphs of a request, so the + // prompt may follow other new input in the same item. + if let Some(start) = paragraph_start(&text, COMPACTION_PROMPT) { + let (before, prompt) = text.split_at(start); + let mut compaction = item.sibling(TrajectoryItemKind::Compaction); + compaction.input = Some(prompt.trim().to_owned()); + let before = before.trim_end(); + if !before.is_empty() { + item.text = Some(before.to_owned()); + self.items.push(item); + } + self.pending_compaction = Some(self.items.len()); + self.items.push(compaction); + continue; + } + let Some(start) = paragraph_start(&text, COMPACTION_HANDOFF) else { + self.items.push(item); + continue; + }; + let (before, handoff) = text.split_at(start); + let before = before.trim_end(); + let replayed = self + .summaries + .iter() + .any(|summary| handoff.trim_end().ends_with(summary.as_str())); + // Without the summary request, the handoff is the only record of + // the compaction. + let recovered = (!replayed).then(|| { + let summary = handoff + .split_once('\n') + .map_or(handoff, |(_, rest)| rest) + .trim() + .to_owned(); + let mut compaction = item.sibling(TrajectoryItemKind::Compaction); + compaction.output = Some(summary.clone()); + compaction.output_event_pk = Some(item.event_pk); + self.summaries.push(summary); + compaction + }); + if !before.is_empty() { + item.text = Some(before.to_owned()); + self.items.push(item); + } + self.items.extend(recovered); + } + } + fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; - if let Some(items) = self.conversation.project(event, stream) { + let projected = self.conversation.project(event, stream); + if let Some(index) = self.pending_compaction.take() { + // The response to the compaction prompt is the handoff summary. + if let Some(summary) = event.text.as_deref().map(str::trim) + && !summary.is_empty() + && let Some(item) = self.items.get_mut(index) + { + item.output = Some(summary.to_owned()); + item.output_event_pk = Some(event.id); + self.summaries.push(summary.to_owned()); + } + } else if let Some(items) = projected { self.items.extend(items); } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); @@ -273,6 +379,18 @@ impl Merger { } } +/// Byte offset of the paragraph that contains `marker`. +fn paragraph_start(text: &str, marker: &str) -> Option { + let position = text.find(marker)?; + Some( + text[..position] + .rfind(PARAGRAPH_SEPARATOR) + .map_or(0, |separator| { + separator.saturating_add(PARAGRAPH_SEPARATOR.len()) + }), + ) +} + /// Identity of a request paragraph. The system fragment's first line carries /// per-request noise, such as a billing header, so it is excluded. fn paragraph_key(index: usize, paragraph: &str) -> u64 { @@ -532,6 +650,117 @@ mod tests { assert_eq!(common_prefix_len("中文a", "中字b"), 3); } + const PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION. Create a \ + handoff summary for another LLM that will resume the task."; + const SUMMARY: &str = "## Goal\n\nFinish the report."; + + fn handoff() -> String { + format!( + "Another language model started to solve this problem and produced a summary \ + of its thinking process. Here is the summary produced by the other language \ + model, use the information in this summary to assist with your own \ + analysis:\n{SUMMARY}" + ) + } + + #[test] + fn compaction_prompt_and_summary_form_one_item_and_the_replay_is_hidden() { + let events = vec![ + event("request", "write the report", json!({})), + event("response", "working", json!({})), + event( + "request", + &format!( + "write the report\n\ndate\n\n{PROMPT}" + ), + json!({}), + ), + event("response", SUMMARY, json!({})), + event( + "request", + &format!( + "cwd\n\n{}", + handoff() + ), + json!({}), + ), + event("response", "continuing", json!({})), + ]; + + let items = build(timeline(events)).items; + + let kinds = items.iter().map(|item| item.kind).collect::>(); + assert_eq!( + kinds, + [ + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User, + TrajectoryItemKind::Compaction, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + ] + ); + // New input sent along with the prompt stays user input. + assert_eq!( + items[2].text.as_deref(), + Some("date") + ); + assert_eq!(items[3].input.as_deref(), Some(PROMPT)); + assert_eq!(items[3].output.as_deref(), Some(SUMMARY)); + // The summary comes from the response, not from the prompt's request. + assert!(items[3].output_event_pk.is_some()); + assert_ne!(items[3].output_event_pk, Some(items[3].event_pk)); + assert_eq!( + items[4].text.as_deref(), + Some("cwd") + ); + } + + #[test] + fn unobserved_compaction_is_recovered_from_its_handoff() { + let events = vec![ + event("request", &format!("resume\n\n{}", handoff()), json!({})), + event("response", "continuing", json!({})), + ]; + + let items = build(timeline(events)).items; + + assert_eq!(items.len(), 3); + assert_eq!(items[0].kind, TrajectoryItemKind::User); + assert_eq!(items[0].text.as_deref(), Some("resume")); + assert_eq!(items[1].kind, TrajectoryItemKind::Compaction); + assert_eq!(items[1].input, None); + assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); + assert_eq!(items[2].kind, TrajectoryItemKind::Assistant); + } + + #[test] + fn snapshot_compaction_is_recognized_by_message() { + let user = json!({"role": "user", "text": "write the report"}); + let prompt = json!({"role": "user", "text": PROMPT}); + let replay = json!({"role": "user", "text": handoff()}); + let conversation = |messages: Vec| json!({"conversation": {"messages": messages}}); + let events = vec![ + event("request", "", conversation(vec![user.clone(), prompt])), + event( + "response", + SUMMARY, + conversation(vec![json!({"role": "assistant", "text": SUMMARY})]), + ), + event("request", "", conversation(vec![user, replay])), + ]; + + let items = build(timeline(events)).items; + + let kinds = items.iter().map(|item| item.kind).collect::>(); + assert_eq!( + kinds, + [TrajectoryItemKind::User, TrajectoryItemKind::Compaction] + ); + assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); + } + fn timeline(events: Vec) -> SessionTimelineResponse { let now = Utc::now(); SessionTimelineResponse { From 04bae7f408e772eaf97333c0ddda5d6833ca6d5e Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 09:20:36 -0700 Subject: [PATCH 08/30] Revert "fix(trajectory): deduplicate replayed snapshots by message identity and order" This reverts commit 9904c1299aa8f8939a831fa3ebceec4f90f8d1c8. --- crates/abyss-backend/src/usage/trajectory.rs | 213 +----------------- .../src/usage/trajectory/conversation.rs | 147 ------------ 2 files changed, 10 insertions(+), 350 deletions(-) delete mode 100644 crates/abyss-backend/src/usage/trajectory/conversation.rs diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 77193e3..29a490b 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -5,10 +5,6 @@ //! new request paragraphs, first-seen tool results, and response text and tool //! calls, with each tool call paired to its result by call identifier. -mod conversation; - -use conversation::ConversationMerger; - use std::{ collections::{HashMap, HashSet, hash_map::DefaultHasher}, hash::{Hash, Hasher}, @@ -161,7 +157,6 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { #[derive(Default)] struct Merger { - conversation: ConversationMerger, items: Vec, seen_paragraphs: HashSet, seen_results: HashSet, @@ -181,22 +176,6 @@ impl Merger { fn request(&mut self, event: &UsageEventResponse) { self.pending_compaction = None; let first_new = self.items.len(); - if let Some(items) = self.conversation.project(event, self.current_stream) { - self.items.extend(items); - } else { - self.legacy_request_text(event); - } - self.mark_compaction(first_new); - let stream = self.current_stream; - for segment in content_segments(&event.metadata) { - if segment_str(segment, "type") == Some("tool_call") { - self.tool_call(event, segment, stream); - } - } - self.request_tools(event, stream); - } - - fn legacy_request_text(&mut self, event: &UsageEventResponse) { let text = event.text.as_deref().unwrap_or_default(); let paragraphs = text.split(PARAGRAPH_SEPARATOR).collect::>(); let first = paragraphs.first().copied().unwrap_or_default(); @@ -245,9 +224,9 @@ impl Merger { } self.last_paragraphs .insert(stream, paragraphs.into_iter().map(str::to_owned).collect()); - } - fn request_tools(&mut self, event: &UsageEventResponse, stream: usize) { + self.mark_compaction(first_new); + for segment in content_segments(&event.metadata) { if segment_str(segment, "type") != Some("tool_result") { continue; @@ -338,7 +317,6 @@ impl Merger { fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; - let projected = self.conversation.project(event, stream); if let Some(index) = self.pending_compaction.take() { // The response to the compaction prompt is the handoff summary. if let Some(summary) = event.text.as_deref().map(str::trim) @@ -349,8 +327,6 @@ impl Merger { item.output_event_pk = Some(event.id); self.summaries.push(summary.to_owned()); } - } else if let Some(items) = projected { - self.items.extend(items); } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); item.text = Some(text.to_owned()); @@ -360,22 +336,15 @@ impl Merger { if segment_str(segment, "type") != Some("tool_call") { continue; } - self.tool_call(event, segment, stream); - } - } - - fn tool_call(&mut self, event: &UsageEventResponse, segment: &Value, stream: usize) { - if segment_str(segment, "call_id").is_some_and(|id| self.calls.contains_key(id)) { - return; - } - let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); - item.name = segment_str(segment, "name").map(str::to_owned); - item.call_id = segment_str(segment, "call_id").map(str::to_owned); - item.input = segment_str(segment, "input").map(str::to_owned); - if let Some(call_id) = &item.call_id { - self.calls.insert(call_id.clone(), self.items.len()); + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); + item.name = segment_str(segment, "name").map(str::to_owned); + item.call_id = segment_str(segment, "call_id").map(str::to_owned); + item.input = segment_str(segment, "input").map(str::to_owned); + if let Some(call_id) = &item.call_id { + self.calls.insert(call_id.clone(), self.items.len()); + } + self.items.push(item); } - self.items.push(item); } } @@ -509,142 +478,6 @@ mod tests { assert!(items[2].continued); } - #[test] - fn snapshots_project_replay_once_and_keep_repeated_user_occurrences() { - let system = json!({"role": "system", "text": "system instructions"}); - let developer = json!({"role": "developer", "text": "developer instructions"}); - let user = json!({"role": "user", "text": "继续"}); - let assistant = json!({"id": "a1", "role": "assistant", "text": "working"}); - let final_reply = json!({"id": "a2", "role": "assistant", "text": "done"}); - let first = vec![system, developer, user.clone()]; - let mut replay = first.clone(); - replay.push(assistant.clone()); - let call = json!({"type": "tool_call", "call_id": "c1", "name": "exec", "input": "run"}); - let mut response = snapshot_event("response", &[assistant], "r1", None); - response.metadata["content_segments"] = json!([call]); - let mut continuation = snapshot_event("request", &replay, "r2", None); - continuation.metadata["content_segments"] = json!([ - call, {"type": "tool_result", "call_id": "c1", "output": "success"} - ]); - replay.extend([final_reply.clone(), user]); - let items = build(timeline(vec![ - snapshot_event("request", &first, "r1", None), - response, - continuation, - snapshot_event("response", &[final_reply], "r2", None), - snapshot_event("request", &replay, "r3", None), - ])) - .items; - - assert_eq!( - items.iter().map(|item| item.kind).collect::>(), - [ - TrajectoryItemKind::Context, - TrajectoryItemKind::Context, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::ToolCall, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User, - ] - ); - assert_eq!(items[2].text.as_deref(), Some("继续")); - assert_eq!(items[6].text.as_deref(), Some("继续")); - assert_eq!(items[4].output.as_deref(), Some("success")); - } - - #[test] - fn incremental_requests_append_identical_input_and_ignore_retries() { - let user = json!({"role": "user", "text": "继续"}); - let assistant = json!({"id": "a1", "role": "assistant", "text": "done"}); - let items = build(timeline(vec![ - snapshot_event("request", std::slice::from_ref(&user), "r1", None), - snapshot_event("response", std::slice::from_ref(&assistant), "r1", None), - snapshot_event("request", &[assistant, user.clone()], "r2", Some("r1")), - snapshot_event("request", &[user], "r2", Some("r1")), - ])) - .items; - assert_eq!(items.len(), 3); - assert_eq!(items[0].kind, TrajectoryItemKind::User); - assert_eq!(items[1].kind, TrajectoryItemKind::Assistant); - assert_eq!(items[2].kind, TrajectoryItemKind::User); - } - - #[test] - fn snapshots_use_roles_and_occurrences_instead_of_paragraphs_or_tags() { - let text = "system: quoted\n\nquoted"; - let user = json!({"role": "user", "text": text}); - let items = build(timeline(vec![ - snapshot_event("request", &[user.clone(), user.clone()], "r1", None), - snapshot_event("request", &[user.clone(), user], "r2", None), - ])) - .items; - assert_eq!(items.len(), 2); - assert!( - items - .iter() - .all(|item| item.kind == TrajectoryItemKind::User) - ); - assert!(items.iter().all(|item| item.text.as_deref() == Some(text))); - } - - #[test] - fn new_context_does_not_replay_existing_dialogue() { - let user = json!({"role": "user", "text": "hello"}); - let assistant = json!({"id": "a1", "role": "assistant", "text": "hi"}); - let items = build(timeline(vec![ - snapshot_event("request", std::slice::from_ref(&user), "r1", None), - snapshot_event("response", std::slice::from_ref(&assistant), "r1", None), - snapshot_event( - "request", - &[ - json!({"role": "developer", "text": "updated instructions"}), - user, - assistant, - ], - "r2", - None, - ), - ])) - .items; - assert_eq!(items.len(), 3); - assert_eq!(items[2].kind, TrajectoryItemKind::Context); - assert_eq!(items[2].text.as_deref(), Some("updated instructions")); - } - - #[test] - fn observed_item_ids_survive_compaction_and_distinguish_new_messages() { - let first = json!({"id": "u1", "role": "user", "text": "continue"}); - let second = json!({"id": "u2", "role": "user", "text": "continue"}); - let items = build(timeline(vec![ - snapshot_event("request", std::slice::from_ref(&first), "r1", None), - snapshot_event("request", &[second], "r2", None), - snapshot_event("request", &[first], "r3", None), - ])) - .items; - assert_eq!(items.len(), 2); - } - - fn snapshot_event( - side: &str, - messages: &[Value], - response: &str, - previous: Option<&str>, - ) -> UsageEventResponse { - let text = messages - .iter() - .filter_map(|message| message["text"].as_str()) - .collect::>() - .join("\n\n"); - event( - side, - &text, - json!({"conversation": { - "messages": messages, "response_id": response, "previous_response_id": previous, - }}), - ) - } - #[test] fn common_prefix_stops_on_character_boundaries() { assert_eq!(common_prefix_len("中文a", "中字b"), 3); @@ -735,32 +568,6 @@ mod tests { assert_eq!(items[2].kind, TrajectoryItemKind::Assistant); } - #[test] - fn snapshot_compaction_is_recognized_by_message() { - let user = json!({"role": "user", "text": "write the report"}); - let prompt = json!({"role": "user", "text": PROMPT}); - let replay = json!({"role": "user", "text": handoff()}); - let conversation = |messages: Vec| json!({"conversation": {"messages": messages}}); - let events = vec![ - event("request", "", conversation(vec![user.clone(), prompt])), - event( - "response", - SUMMARY, - conversation(vec![json!({"role": "assistant", "text": SUMMARY})]), - ), - event("request", "", conversation(vec![user, replay])), - ]; - - let items = build(timeline(events)).items; - - let kinds = items.iter().map(|item| item.kind).collect::>(); - assert_eq!( - kinds, - [TrajectoryItemKind::User, TrajectoryItemKind::Compaction] - ); - assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); - } - fn timeline(events: Vec) -> SessionTimelineResponse { let now = Utc::now(); SessionTimelineResponse { diff --git a/crates/abyss-backend/src/usage/trajectory/conversation.rs b/crates/abyss-backend/src/usage/trajectory/conversation.rs deleted file mode 100644 index a79d90a..0000000 --- a/crates/abyss-backend/src/usage/trajectory/conversation.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Projects role-aware snapshots without changing stored request or response text. -//! -//! Match occurrences in order, rather than deduplicating strings globally: a -//! second identical user message is still a new message. Provider item ids are -//! used when available, and incremental requests append their new user input. - -use std::collections::{HashMap, HashSet}; - -use serde::Deserialize; - -use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; - -#[derive(Clone, Deserialize)] -struct Message { - #[serde(default)] - id: Option, - role: Role, - text: String, -} - -#[derive(Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "snake_case")] -enum Role { - System, - Developer, - User, - Assistant, - Reasoning, -} - -impl Role { - const fn kind(self) -> TrajectoryItemKind { - match self { - Self::System | Self::Developer => TrajectoryItemKind::Context, - Self::User => TrajectoryItemKind::User, - Self::Assistant | Self::Reasoning => TrajectoryItemKind::Assistant, - } - } - - const fn is_context(self) -> bool { - matches!(self, Self::System | Self::Developer) - } -} - -impl Message { - fn matches(&self, other: &Self) -> bool { - self.role == other.role - && self.text == other.text - && match (&self.id, &other.id) { - (Some(left), Some(right)) => left == right, - _ => true, - } - } -} - -#[derive(Deserialize)] -struct Snapshot { - messages: Vec, - response_id: Option, - previous_response_id: Option, -} - -#[derive(Default)] -pub(super) struct ConversationMerger { - history: Vec, - contexts: Vec, - identified: HashMap, - /// The same provider response may be observed again after a retry. - requests: HashSet, - responses: HashSet, -} - -impl ConversationMerger { - /// None keeps the existing legacy projection for events without snapshots. - pub(super) fn project( - &mut self, - event: &UsageEventResponse, - stream: usize, - ) -> Option> { - let snapshot: Snapshot = - serde_json::from_value(event.metadata.get("conversation")?.clone()).ok()?; - let is_request = event.event_type == "request"; - if let Some(id) = snapshot.response_id { - let seen = if is_request { - &mut self.requests - } else { - &mut self.responses - }; - if !seen.insert(id) { - return Some(Vec::new()); - } - } - let incremental = snapshot.previous_response_id.is_some(); - let mut cursor = 0; - let mut history = Vec::new(); - let mut added = Vec::new(); - for message in snapshot.messages { - let known_id = message - .id - .as_ref() - .and_then(|id| self.identified.get(id)) - .is_some_and(|earlier| earlier.matches(&message)); - let already_seen = if message.role.is_context() { - let known = self - .contexts - .iter() - .any(|earlier| earlier.matches(&message)); - if !known { - self.contexts.push(message.clone()); - } - known - } else if is_request && !incremental { - let found = self.history[cursor..] - .iter() - .position(|earlier| earlier.matches(&message)); - if let Some(offset) = found { - cursor = cursor.saturating_add(offset).saturating_add(1); - } - history.push(message.clone()); - known_id || found.is_some() - } else { - // Incremental user messages represent new occurrences even if - // they say the same thing as an earlier message. Replayed - // assistant items can still refer to an observed response. - let replay = is_request - && message.role != Role::User - && self.history.iter().any(|earlier| earlier.matches(&message)); - if !known_id && !replay { - self.history.push(message.clone()); - } - known_id || replay - }; - if let Some(id) = &message.id { - self.identified.insert(id.clone(), message.clone()); - } - if !already_seen && !message.text.trim().is_empty() { - let mut item = TrajectoryItem::new(message.role.kind(), stream, event); - item.text = Some(message.text); - added.push(item); - } - } - if is_request && !incremental { - self.history = history; - } - Some(added) - } -} From d27c8eb1f472933e23508f96d88272adeefef227 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 10:01:08 -0700 Subject: [PATCH 09/30] fix(trajectory): project archived bodies without changing stored records --- crates/abyss-backend/src/api.rs | 2 +- .../src/storage/sqlite/repository.rs | 41 ++ crates/abyss-backend/src/usage/trajectory.rs | 164 +++++++- .../src/usage/trajectory/raw_body.rs | 351 ++++++++++++++++++ 4 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 crates/abyss-backend/src/usage/trajectory/raw_body.rs diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index 6d36468..a386557 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -30,7 +30,7 @@ use crate::{ }, }; -const MAX_INGEST_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024; +const MAX_INGEST_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024; #[derive(Clone)] /// Cloneable dependencies and request limits shared by all Axum handlers. diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index 6600c5c..32ed061 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -1442,6 +1442,47 @@ mod tests { ); } + #[test] + fn raw_body_archive_survives_ingest_retries_and_trajectory_projection() { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + use sha2::Digest as _; + let original = + b" {\"input\":[{\"role\":\"user\",\"content\":\"hello\"}],\"unknown\": [1,1]} \r\n"; + let mut database = database(); + let mut request = request_with_text("hello"); + request.events[0].event_type = crate::usage::UsageEventType::Request; + request.events[0].metadata["raw_body"] = json!({"transport":"http", "parts":[{ + "content_base64": STANDARD.encode(original), "byte_size": original.len(), "truncated": false, + "sha256": hex::encode(sha2::Sha256::digest(original)) + }]}); + let expected = request.events[0].metadata.clone(); + ingest_events(&mut database, &request, OWNER_ID, 100).unwrap(); + ingest_events(&mut database, &request, OWNER_ID, 100).unwrap(); + let session_pk = llm_usage_events::table + .select(llm_usage_events::session_pk) + .first::(&mut database) + .unwrap(); + let session_pk = Uuid::parse_str(&session_pk).unwrap(); + let timeline = super::session_timeline(&mut database, OWNER_ID, session_pk).unwrap(); + let stored = &timeline.turns[0].events[0].metadata; + assert_eq!(*stored, expected); + assert_eq!( + STANDARD + .decode( + stored["raw_body"]["parts"][0]["content_base64"] + .as_str() + .unwrap() + ) + .unwrap(), + original + ); + let trajectory = crate::usage::trajectory::build(timeline); + assert_eq!(trajectory.items[0].text.as_deref(), Some("hello")); + let after = super::session_timeline(&mut database, OWNER_ID, session_pk).unwrap(); + assert_eq!(after.turns[0].events.len(), 1); + assert_eq!(after.turns[0].events[0].metadata, expected); + } + fn request_with_text(text: &str) -> IngestEventsRequest { serde_json::from_value(json!({ "events": [{ diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 29a490b..05af664 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -5,6 +5,10 @@ //! new request paragraphs, first-seen tool results, and response text and tool //! calls, with each tool call paired to its result by call identifier. +mod raw_body; + +use raw_body::RawProjection; + use std::{ collections::{HashMap, HashSet, hash_map::DefaultHasher}, hash::{Hash, Hasher}, @@ -157,6 +161,7 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { #[derive(Default)] struct Merger { + raw: RawProjection, items: Vec, seen_paragraphs: HashSet, seen_results: HashSet, @@ -176,6 +181,21 @@ impl Merger { fn request(&mut self, event: &UsageEventResponse) { self.pending_compaction = None; let first_new = self.items.len(); + if let Some(context) = RawProjection::context(event) { + let key = paragraph_key(0, &context); + let next = self.streams.len(); + self.current_stream = *self.streams.entry(key).or_insert(next); + } + if let Some(items) = self.raw.project(event, self.current_stream) { + self.items.extend(items); + } else { + self.legacy_request_text(event); + } + self.mark_compaction(first_new); + self.request_tools(event); + } + + fn legacy_request_text(&mut self, event: &UsageEventResponse) { let text = event.text.as_deref().unwrap_or_default(); let paragraphs = text.split(PARAGRAPH_SEPARATOR).collect::>(); let first = paragraphs.first().copied().unwrap_or_default(); @@ -224,8 +244,10 @@ impl Merger { } self.last_paragraphs .insert(stream, paragraphs.into_iter().map(str::to_owned).collect()); + } - self.mark_compaction(first_new); + fn request_tools(&mut self, event: &UsageEventResponse) { + let stream = self.current_stream; for segment in content_segments(&event.metadata) { if segment_str(segment, "type") != Some("tool_result") { @@ -317,6 +339,7 @@ impl Merger { fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; + let projected = self.raw.project(event, stream); if let Some(index) = self.pending_compaction.take() { // The response to the compaction prompt is the handoff summary. if let Some(summary) = event.text.as_deref().map(str::trim) @@ -327,6 +350,8 @@ impl Merger { item.output_event_pk = Some(event.id); self.summaries.push(summary.to_owned()); } + } else if let Some(items) = projected { + self.items.extend(items); } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); item.text = Some(text.to_owned()); @@ -483,6 +508,143 @@ mod tests { assert_eq!(common_prefix_len("中文a", "中字b"), 3); } + fn raw_event(side: &str, body: &str) -> UsageEventResponse { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + event( + side, + "legacy fallback", + json!({"raw_body": {"transport": "http", "parts": [{ + "content_base64": STANDARD.encode(body.as_bytes()), "truncated": false, + "byte_size": body.len(), "content_type": "application/json" + }]}}), + ) + } + + #[test] + fn raw_archive_projection_keeps_roles_and_repeated_user_occurrences() { + let first = r#" {"instructions":"system instructions","input":[{"role":"user","content":"继续"}],"unknown":[1,1]} "#; + let response = r#"{"output":[{"id":"a1","role":"assistant","content":[{"type":"output_text","text":"working"}]}]}"#; + let replay = r#"{"instructions":"system instructions","input":[{"role":"user","content":"继续"},{"id":"a1","role":"assistant","content":[{"type":"output_text","text":"working"}]},{"role":"user","content":"继续"}]}"#; + let events = vec![ + raw_event("request", first), + raw_event("response", response), + raw_event("request", replay), + raw_event("request", replay), + ]; + let metadata_before = events + .iter() + .map(|e| e.metadata.clone()) + .collect::>(); + let mut merger = Merger::default(); + for e in &events { + if e.event_type == "request" { + merger.request(e); + } else { + merger.response(e); + } + } + assert_eq!( + merger.items.iter().map(|i| i.kind).collect::>(), + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User + ] + ); + assert_eq!(merger.items[1].text.as_deref(), Some("继续")); + assert_eq!(merger.items[3].text.as_deref(), Some("继续")); + assert_eq!( + metadata_before, + events + .iter() + .map(|e| e.metadata.clone()) + .collect::>() + ); + } + + #[test] + fn streamed_anthropic_response_is_not_replayed_as_user_input() { + let response = "event: content_block_start\r\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\r\n\r\nevent: content_block_delta\r\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi\"}}\r\n\r\n"; + let items = build(timeline(vec![ + raw_event("request", r#"{"system":"instructions","messages":[{"role":"user","content":"hello"}]}"#), + raw_event("response", response), + raw_event("request", r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"assistant","content":"hi"},{"role":"user","content":"hello"}]}"#), + ])).items; + assert_eq!( + items.iter().map(|i| i.kind).collect::>(), + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User + ] + ); + assert_eq!(items[2].text.as_deref(), Some("hi")); + } + + #[test] + fn incremental_requests_keep_identical_new_user_messages() { + let items = build(timeline(vec![ + raw_event( + "request", + r#"{"instructions":"instructions","input":[{"role":"user","content":"continue"}]}"#, + ), + raw_event("response", r#"{"output_text":"working"}"#), + raw_event( + "request", + r#"{"previous_response_id":"r1","input":[{"role":"user","content":"continue"}]}"#, + ), + ])) + .items; + assert_eq!( + items.iter().map(|i| i.kind).collect::>(), + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + + #[test] + fn raw_anthropic_request_retains_system_and_assistant_roles() { + let body = r#"{"system":[{"type":"text","text":"instructions"}],"messages":[{"role":"user","content":"hello"},{"role":"assistant","content":[{"type":"text","text":"hi"}]},{"role":"user","content":"hello"}]}"#; + let items = build(timeline(vec![ + raw_event("request", body), + raw_event("request", body), + ])) + .items; + assert_eq!( + items.iter().map(|i| i.kind).collect::>(), + [ + TrajectoryItemKind::Context, + TrajectoryItemKind::User, + TrajectoryItemKind::Assistant, + TrajectoryItemKind::User + ] + ); + } + + #[test] + fn raw_compaction_keeps_existing_summary_presentation() { + let request = json!({"input": [{"role": "user", "content": "write report"}, {"role": "user", "content": PROMPT}]}).to_string(); + let replay = json!({"input": [{"role": "user", "content": "write report"}, {"role": "user", "content": handoff()}]}).to_string(); + let items = build(timeline(vec![ + raw_event("request", &request), + event("response", SUMMARY, json!({})), + raw_event("request", &replay), + ])) + .items; + assert_eq!( + items.iter().map(|i| i.kind).collect::>(), + [TrajectoryItemKind::User, TrajectoryItemKind::Compaction] + ); + assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); + } + const PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION. Create a \ handoff summary for another LLM that will resume the task."; const SUMMARY: &str = "## Goal\n\nFinish the report."; diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs new file mode 100644 index 0000000..237ab4b --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory/raw_body.rs @@ -0,0 +1,351 @@ +//! Read-only projection of archived provider bodies; the archive is never edited. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde_json::Value; + +use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; + +#[derive(Clone)] +struct Message { + id: Option, + role: String, + text: String, +} + +impl Message { + fn matches(&self, other: &Self) -> bool { + self.role == other.role + && self.text == other.text + && match (&self.id, &other.id) { + (Some(left), Some(right)) => left == right, + _ => true, + } + } + + fn kind(&self) -> TrajectoryItemKind { + match self.role.as_str() { + "system" | "developer" => TrajectoryItemKind::Context, + "user" => TrajectoryItemKind::User, + _ => TrajectoryItemKind::Assistant, + } + } +} + +#[derive(Default)] +pub(super) struct RawProjection { + streams: HashMap, +} + +#[derive(Default)] +struct StreamHistory { + history: Vec, + contexts: HashSet<(String, String)>, +} + +impl RawProjection { + pub(super) fn context(event: &UsageEventResponse) -> Option { + let values = bodies(event)?; + let payload = request_payload(&values)?; + if payload + .get("previous_response_id") + .is_some_and(Value::is_string) + && payload.get("instructions").is_none() + && payload.get("system").is_none() + { + return None; + } + Some( + request_messages(payload) + .into_iter() + .filter(|m| m.kind() == TrajectoryItemKind::Context) + .map(|m| format!("{}: {}", m.role, m.text)) + .collect::>() + .join("\n\n"), + ) + } + + pub(super) fn project( + &mut self, + event: &UsageEventResponse, + stream: usize, + ) -> Option> { + let values = bodies(event)?; + let request = event.event_type == "request"; + let (messages, incremental) = if request { + let payload = request_payload(&values)?; + ( + request_messages(payload), + payload + .get("previous_response_id") + .is_some_and(Value::is_string), + ) + } else { + let mut messages = response_messages(&values); + if messages.is_empty() + && let Some(text) = event.text.as_ref().filter(|v| !v.is_empty()) + { + messages.push(Message { + id: None, + role: "assistant".to_owned(), + text: text.clone(), + }); + } + (messages, false) + }; + // Incomplete/unrecognized response formats still have the older parsed + // text projection. Archival success must not depend on this parser. + if !request && messages.is_empty() { + return None; + } + let state = self.streams.entry(stream).or_default(); + let mut cursor = 0; + let mut next = Vec::new(); + let mut added = Vec::new(); + for message in messages { + let context = message.kind() == TrajectoryItemKind::Context; + let known = if context { + !state + .contexts + .insert((message.role.clone(), message.text.clone())) + } else if request && !incremental { + let found = state.history[cursor..] + .iter() + .position(|old| old.matches(&message)); + if let Some(offset) = found { + cursor = cursor.saturating_add(offset).saturating_add(1); + } + found.is_some() + } else if request { + state.history.iter().any(|old| { + old.matches(&message) && (message.id.is_some() || message.role != "user") + }) + } else { + false + }; + if !context { + if request && !incremental { + next.push(message.clone()); + } else if !known { + state.history.push(message.clone()); + } + } + if !known && !message.text.is_empty() { + let mut item = TrajectoryItem::new(message.kind(), stream, event); + item.text = Some(message.text); + added.push(item); + } + } + if request && !incremental { + state.history = next; + } + Some(added) + } +} + +fn request_payload(values: &[Value]) -> Option<&Value> { + values + .iter() + .map(|v| v.get("response").unwrap_or(v)) + .find(|v| v.get("input").is_some() || v.get("messages").is_some()) +} + +/// Decode bytes only for projection. The original base64, whitespace, unknown +/// fields, and SSE events remain untouched in event metadata. +fn bodies(event: &UsageEventResponse) -> Option> { + let parts = event.metadata.get("raw_body")?.get("parts")?.as_array()?; + let mut values = Vec::new(); + for part in parts { + if part.get("truncated").and_then(Value::as_bool) != Some(false) + || part + .get("content_encoding") + .and_then(Value::as_str) + .is_some_and(|v| v != "identity") + { + return None; + } + let bytes = STANDARD + .decode(part.get("content_base64")?.as_str()?) + .ok()?; + if let Ok(value) = serde_json::from_slice(&bytes) { + values.push(value); + } else { + let text = std::str::from_utf8(&bytes).ok()?; + let mut data = String::new(); + for line in text.lines().chain(std::iter::once("")) { + if line.is_empty() { + if let Ok(value) = serde_json::from_str(&data) { + values.push(value); + } + data.clear(); + } else if let Some(line) = line.strip_prefix("data:") { + if !data.is_empty() { + data.push('\n'); + } + data.push_str(line.strip_prefix(' ').unwrap_or(line)); + } + } + } + } + Some(values) +} + +fn request_messages(payload: &Value) -> Vec { + let mut result = Vec::new(); + for (key, role) in [("instructions", "developer"), ("system", "system")] { + if let Some(value) = payload.get(key) { + result.push(Message { + id: None, + role: role.to_owned(), + text: text(value), + }); + } + } + if let Some(input) = payload.get("input").or_else(|| payload.get("messages")) { + if let Some(items) = input.as_array() { + result.extend(items.iter().filter_map(|v| message(v, "user"))); + } else { + result.extend(message(input, "user")); + } + } + result +} + +fn message(value: &Value, default_role: &str) -> Option { + let kind = value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + let role = value + .get("role") + .and_then(Value::as_str) + .unwrap_or(default_role); + if role == "tool" || tool(kind) { + return None; + } + let (role, content) = if kind == "reasoning" { + ("reasoning", value.get("summary")?) + } else { + (role, value.get("content").unwrap_or(value)) + }; + Some(Message { + id: value.get("id").and_then(Value::as_str).map(str::to_owned), + role: role.to_owned(), + text: text(content), + }) +} + +fn text(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + Value::Array(values) => values + .iter() + .map(text) + .filter(|v| !v.is_empty()) + .collect::>() + .join("\n"), + Value::Object(obj) => { + let kind = obj.get("type").and_then(Value::as_str).unwrap_or_default(); + if tool(kind) { + return String::new(); + } + obj.get("text") + .or_else(|| obj.get("content")) + .or_else(|| obj.get("thinking")) + .or_else(|| obj.get("output_text")) + .map(text) + .unwrap_or_default() + } + _ => String::new(), + } +} + +fn tool(kind: &str) -> bool { + matches!(kind, "tool_use" | "tool_result" | "tool_call") + || kind.ends_with("_call") + || kind.ends_with("_call_output") +} + +fn response_messages(values: &[Value]) -> Vec { + let mut items = BTreeMap::new(); + let mut anthropic_blocks = BTreeMap::::new(); + let mut chat_text = String::new(); + for value in values { + let kind = value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + let index = value.get("index").and_then(Value::as_u64).unwrap_or(0); + if kind == "content_block_start" { + if let Some(block) = value.get("content_block") { + anthropic_blocks.insert(index, text(block)); + } + } else if kind == "content_block_delta" + && let Some(delta) = value.get("delta") + { + anthropic_blocks + .entry(index) + .or_default() + .push_str(&text(delta)); + } + if let Some(delta) = value + .pointer("/choices/0/delta/content") + .and_then(Value::as_str) + { + chat_text.push_str(delta); + } + let response = value.get("response").unwrap_or(value); + if let Some(output) = response.get("output").and_then(Value::as_array) { + for (index, value) in output.iter().enumerate() { + if let Some(message) = message(value, "assistant") { + items.insert(u64::try_from(index).unwrap_or(u64::MAX), message); + } + } + } else if value.get("type").and_then(Value::as_str) == Some("response.output_item.done") { + if let Some(message) = value.get("item").and_then(|v| message(v, "assistant")) { + items.insert( + value + .get("output_index") + .and_then(Value::as_u64) + .unwrap_or(0), + message, + ); + } + } else if let Some(message) = value + .get("choices") + .and_then(Value::as_array) + .and_then(|v| v.first()) + .and_then(|v| v.get("message")) + .and_then(|v| message(v, "assistant")) + { + items.insert(0, message); + } else if (value.get("content").is_some() || value.get("output_text").is_some()) + && let Some(message) = message(value, "assistant") + { + items.insert(0, message); + } + } + if items.is_empty() { + let text = if anthropic_blocks.is_empty() { + chat_text + } else { + anthropic_blocks + .into_values() + .filter(|s| !s.is_empty()) + .collect::>() + .join("\n") + }; + if !text.is_empty() { + items.insert( + 0, + Message { + id: None, + role: "assistant".to_owned(), + text, + }, + ); + } + } + items.into_values().collect() +} From 00a5be4ee8dc44f7fe1aca950b026bb65471cd1f Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 10:56:42 -0700 Subject: [PATCH 10/30] fix(trajectory): remove text-based role guessing and prefix trimming --- crates/abyss-backend/src/usage/trajectory.rs | 174 +++++++----------- .../src/usage/trajectory/raw_body.rs | 7 +- 2 files changed, 70 insertions(+), 111 deletions(-) diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 05af664..fefcbc4 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -2,7 +2,7 @@ //! //! Collectors record each provider request verbatim, so every request repeats //! the conversation so far. The trajectory keeps only what each event adds: -//! new request paragraphs, first-seen tool results, and response text and tool +//! role-aware messages, first-seen tool results, and response text and tool //! calls, with each tool call paired to its result by call identifier. mod raw_body; @@ -23,10 +23,6 @@ use super::{SessionInfo, SessionTimelineResponse, UsageEventResponse}; /// Collectors join request fragments with this separator. const PARAGRAPH_SEPARATOR: &str = "\n\n"; -/// Collectors prefix the system prompt fragment with this marker. -const SYSTEM_PREFIX: &str = "system: "; -/// Shared prefix length above which a changed paragraph counts as grown. -const GROWN_PARAGRAPH_MIN_PREFIX_BYTES: usize = 200; /// Codex opens the request that asks the model for a context handoff summary /// with this prompt when it compacts its context. const COMPACTION_PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION."; @@ -48,9 +44,11 @@ pub struct SessionTrajectoryResponse { #[serde(rename_all = "snake_case")] /// Kind of one trajectory item. pub enum TrajectoryItemKind { - /// First request content of a stream, starting with its system prompt. + /// Request content whose message role is unavailable. + Request, + /// Explicit system or developer message. Context, - /// Request paragraphs not seen earlier in the session. + /// Explicit user message not seen earlier in the session. User, /// Response text. Assistant, @@ -163,10 +161,8 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { struct Merger { raw: RawProjection, items: Vec, - seen_paragraphs: HashSet, seen_results: HashSet, streams: HashMap, - last_paragraphs: HashMap>, /// Tool call item index by call identifier. calls: HashMap, /// Stream of the most recent request; responses belong to it. @@ -182,68 +178,26 @@ impl Merger { self.pending_compaction = None; let first_new = self.items.len(); if let Some(context) = RawProjection::context(event) { - let key = paragraph_key(0, &context); + let key = hash_str(&context); let next = self.streams.len(); self.current_stream = *self.streams.entry(key).or_insert(next); } if let Some(items) = self.raw.project(event, self.current_stream) { self.items.extend(items); } else { - self.legacy_request_text(event); + self.unclassified_request(event); } self.mark_compaction(first_new); self.request_tools(event); } - fn legacy_request_text(&mut self, event: &UsageEventResponse) { - let text = event.text.as_deref().unwrap_or_default(); - let paragraphs = text.split(PARAGRAPH_SEPARATOR).collect::>(); - let first = paragraphs.first().copied().unwrap_or_default(); - let stream_key = if first.starts_with(SYSTEM_PREFIX) { - paragraph_key(0, first) - } else { - 0 - }; - let next_stream = self.streams.len(); - let stream = *self.streams.entry(stream_key).or_insert(next_stream); - self.current_stream = stream; - - let previous = self.last_paragraphs.remove(&stream).unwrap_or_default(); - let mut added = Vec::new(); - let mut continued = false; - for (index, paragraph) in paragraphs.iter().enumerate() { - if paragraph.trim().is_empty() - || !self.seen_paragraphs.insert(paragraph_key(index, paragraph)) - { - continue; - } - // A paragraph that grew since the stream's previous request, such - // as an embedded transcript, contributes only its new tail. - let shared = previous - .iter() - .map(|earlier| common_prefix_len(paragraph, earlier)) - .max() - .unwrap_or(0); - if shared >= GROWN_PARAGRAPH_MIN_PREFIX_BYTES { - added.push(¶graph[shared..]); - continued = true; - } else { - added.push(paragraph); - } - } - if !added.is_empty() { - let kind = if added[0].starts_with(SYSTEM_PREFIX) { - TrajectoryItemKind::Context - } else { - TrajectoryItemKind::User - }; - let mut item = TrajectoryItem::new(kind, stream, event); - item.text = Some(added.join(PARAGRAPH_SEPARATOR)); - item.continued = continued; + fn unclassified_request(&mut self, event: &UsageEventResponse) { + if let Some(text) = event.text.as_ref().filter(|text| !text.is_empty()) { + let mut item = + TrajectoryItem::new(TrajectoryItemKind::Request, self.current_stream, event); + item.text = Some(text.clone()); self.items.push(item); } - self.last_paragraphs - .insert(stream, paragraphs.into_iter().map(str::to_owned).collect()); } fn request_tools(&mut self, event: &UsageEventResponse) { @@ -385,34 +339,12 @@ fn paragraph_start(text: &str, marker: &str) -> Option { ) } -/// Identity of a request paragraph. The system fragment's first line carries -/// per-request noise, such as a billing header, so it is excluded. -fn paragraph_key(index: usize, paragraph: &str) -> u64 { - if index == 0 && paragraph.starts_with(SYSTEM_PREFIX) { - return hash_str(paragraph.split_once('\n').map_or("", |(_, rest)| rest)); - } - hash_str(paragraph) -} - fn hash_str(value: &str) -> u64 { let mut hasher = DefaultHasher::new(); value.hash(&mut hasher); hasher.finish() } -/// Length in bytes of the shared prefix, ending on a character boundary. -fn common_prefix_len(left: &str, right: &str) -> usize { - let mut len = left - .bytes() - .zip(right.bytes()) - .take_while(|(a, b)| a == b) - .count(); - while !left.is_char_boundary(len) { - len = len.saturating_sub(1); - } - len -} - fn content_segments(metadata: &Value) -> &[Value] { metadata .get("content_segments") @@ -432,7 +364,7 @@ mod tests { use crate::usage::TurnTimeline; #[test] - fn keeps_only_new_content_and_pairs_tool_results() { + fn unclassified_requests_keep_every_observation_and_pair_tool_results() { let system = "system: header cch=1\nYou are an agent."; let events = vec![ event("request", &format!("{system}\n\nfirst prompt"), json!({})), @@ -466,19 +398,23 @@ mod tests { assert_eq!( kinds, [ - TrajectoryItemKind::Context, + TrajectoryItemKind::Request, TrajectoryItemKind::ToolCall, + TrajectoryItemKind::Request, TrajectoryItemKind::Assistant, - TrajectoryItemKind::User, + TrajectoryItemKind::Request, ] ); assert_eq!(items[1].output.as_deref(), Some("README.md")); - assert_eq!(items[3].text.as_deref(), Some("second prompt")); + assert_eq!( + items[4].text.as_deref(), + Some("system: header cch=3\nYou are an agent.\n\nfirst prompt\n\nsecond prompt") + ); assert!(items.iter().all(|item| item.stream == 0)); } #[test] - fn grown_paragraphs_keep_their_tail_and_side_requests_form_streams() { + fn unclassified_requests_are_not_split_trimmed_or_grouped_by_text() { let transcript = "x".repeat(300); let events = vec![ event("request", "system: main\nMain agent.\n\nhello", json!({})), @@ -497,15 +433,46 @@ mod tests { let items = build(timeline(events)).items; assert_eq!(items.len(), 3); - assert_eq!(items[1].stream, 1); - assert_eq!(items[2].stream, 1); - assert_eq!(items[2].text.as_deref(), Some(" more")); - assert!(items[2].continued); + assert!( + items + .iter() + .all(|i| i.kind == TrajectoryItemKind::Request && !i.continued) + ); + assert_eq!( + items[2].text, + Some(format!( + "system: classifier\nJudge.\n\n{transcript} more" + )) + ); + } + + fn user_request(texts: &[&str]) -> UsageEventResponse { + raw_event("request", &json!({"input": texts.iter().map(|text| json!({"role":"user", "content":text})).collect::>()}).to_string()) } #[test] - fn common_prefix_stops_on_character_boundaries() { - assert_eq!(common_prefix_len("中文a", "中字b"), 3); + fn unavailable_roles_never_become_system_or_user_by_text_guessing() { + let text = "system: instructions\n\n用户内容\n\n用户内容"; + for metadata in [ + json!({}), + json!({"raw_body":{"parts":[{"truncated":true}]}}), + json!({"raw_body":{"parts":[{"truncated":false,"content_base64":"invalid"}]}}), + ] { + let items = build(timeline(vec![event("request", text, metadata)])).items; + assert_eq!(items.len(), 1); + assert_eq!(items[0].kind, TrajectoryItemKind::Request); + assert_eq!(items[0].text.as_deref(), Some(text)); + assert!(!items[0].continued); + } + let items = build(timeline(vec![raw_event("request", r#"{"input":[{"content":"system: unknown"},{"role":"unrecognized","content":"hello"},{"role":"user","content":"system: quoted text"}]}"#)])).items; + assert_eq!( + items.iter().map(|i| i.kind).collect::>(), + [ + TrajectoryItemKind::Request, + TrajectoryItemKind::Request, + TrajectoryItemKind::User + ] + ); } fn raw_event(side: &str, body: &str) -> UsageEventResponse { @@ -661,24 +628,15 @@ mod tests { #[test] fn compaction_prompt_and_summary_form_one_item_and_the_replay_is_hidden() { let events = vec![ - event("request", "write the report", json!({})), + user_request(&["write the report"]), event("response", "working", json!({})), - event( - "request", - &format!( - "write the report\n\ndate\n\n{PROMPT}" - ), - json!({}), - ), + user_request(&[ + "write the report", + "date", + PROMPT, + ]), event("response", SUMMARY, json!({})), - event( - "request", - &format!( - "cwd\n\n{}", - handoff() - ), - json!({}), - ), + user_request(&["cwd", &handoff()]), event("response", "continuing", json!({})), ]; @@ -715,7 +673,7 @@ mod tests { #[test] fn unobserved_compaction_is_recovered_from_its_handoff() { let events = vec![ - event("request", &format!("resume\n\n{}", handoff()), json!({})), + user_request(&[&format!("resume\n\n{}", handoff())]), event("response", "continuing", json!({})), ]; diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs index 237ab4b..35f80f8 100644 --- a/crates/abyss-backend/src/usage/trajectory/raw_body.rs +++ b/crates/abyss-backend/src/usage/trajectory/raw_body.rs @@ -28,7 +28,8 @@ impl Message { match self.role.as_str() { "system" | "developer" => TrajectoryItemKind::Context, "user" => TrajectoryItemKind::User, - _ => TrajectoryItemKind::Assistant, + "assistant" | "reasoning" => TrajectoryItemKind::Assistant, + _ => TrajectoryItemKind::Request, } } } @@ -204,9 +205,9 @@ fn request_messages(payload: &Value) -> Vec { } if let Some(input) = payload.get("input").or_else(|| payload.get("messages")) { if let Some(items) = input.as_array() { - result.extend(items.iter().filter_map(|v| message(v, "user"))); + result.extend(items.iter().filter_map(|v| message(v, "request"))); } else { - result.extend(message(input, "user")); + result.extend(message(input, "request")); } } result From 2e72a571b3370d493832078e63c2ec41fd0d31c4 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 30 Sep 2026 11:05:53 -0700 Subject: [PATCH 11/30] feat: match Codex naming threads with the task sessions they name Codex names a task from a separate thread whose requests carry no link to the task (verified on 169 captured naming requests across Codex 0.154-0.159). The naming request quotes the task's user message, and Codex starts the naming turn right after the task turn. A task session is matched when its turn ID (UUIDv7) starts within 60 s of the naming turn on the same device and its latest user message opens with the quoted prompt. Only a single candidate is matched; a later second candidate withdraws the match. The title is then recorded for the task session, and the naming thread keeps a link to it. Co-Authored-By: Claude Opus 5.5 --- .../20261001000000_codex_title_links/down.sql | 3 + .../20261001000000_codex_title_links/up.sql | 6 + .../20261001000000_codex_title_links/down.sql | 4 + .../20261001000000_codex_title_links/up.sql | 7 + crates/abyss-backend/src/db/schema.rs | 3 + .../src/storage/postgres/repository.rs | 11 +- .../src/storage/postgres/session_titles.rs | 437 +++++++++++--- .../src/storage/sqlite/repository.rs | 11 +- .../src/storage/sqlite/schema.rs | 3 + .../src/storage/sqlite/session_titles.rs | 540 +++++++++++++++--- crates/abyss-backend/src/usage/mod.rs | 2 + .../abyss-backend/src/usage/session_title.rs | 99 ++++ crates/abyss-backend/src/usage/trajectory.rs | 1 + 13 files changed, 968 insertions(+), 159 deletions(-) create mode 100644 crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql create mode 100644 crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql create mode 100644 crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql create mode 100644 crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql diff --git a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql new file mode 100644 index 0000000..9603ac4 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql @@ -0,0 +1,3 @@ +ALTER TABLE session_titles DROP COLUMN parent_session_pk; +ALTER TABLE session_titles DROP COLUMN naming_prompt; +ALTER TABLE session_titles DROP COLUMN naming_turn_id; diff --git a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql new file mode 100644 index 0000000..ee84ecd --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql @@ -0,0 +1,6 @@ +-- Codex names a task from a separate naming thread. naming_turn_id and +-- naming_prompt keep what that thread's request shows about the task it names, +-- and parent_session_pk records the session it was matched to. +ALTER TABLE session_titles ADD COLUMN naming_turn_id TEXT; +ALTER TABLE session_titles ADD COLUMN naming_prompt TEXT; +ALTER TABLE session_titles ADD COLUMN parent_session_pk TEXT REFERENCES agent_sessions(id) ON DELETE SET NULL; diff --git a/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql b/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql new file mode 100644 index 0000000..34b83c0 --- /dev/null +++ b/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql @@ -0,0 +1,4 @@ +ALTER TABLE session_titles + DROP COLUMN parent_session_pk, + DROP COLUMN naming_prompt, + DROP COLUMN naming_turn_id; diff --git a/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql b/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql new file mode 100644 index 0000000..6e56246 --- /dev/null +++ b/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql @@ -0,0 +1,7 @@ +-- Codex names a task from a separate naming thread. naming_turn_id and +-- naming_prompt keep what that thread's request shows about the task it names, +-- and parent_session_pk records the session it was matched to. +ALTER TABLE session_titles + ADD COLUMN naming_turn_id text, + ADD COLUMN naming_prompt text, + ADD COLUMN parent_session_pk uuid REFERENCES agent_sessions(id) ON DELETE SET NULL; diff --git a/crates/abyss-backend/src/db/schema.rs b/crates/abyss-backend/src/db/schema.rs index 5b8cedd..1adb464 100644 --- a/crates/abyss-backend/src/db/schema.rs +++ b/crates/abyss-backend/src/db/schema.rs @@ -163,6 +163,9 @@ diesel::table! { title_observed_at -> Nullable, purpose -> Nullable, updated_at -> Timestamptz, + naming_turn_id -> Nullable, + naming_prompt -> Nullable, + parent_session_pk -> Nullable, } } diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index f0439a4..a378a9b 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -377,8 +377,7 @@ pub fn session_timeline( )); } let mut timeline = assemble_timeline(session, turns, events_by_turn); - (timeline.session.title, timeline.session.purpose) = - session_titles::load(connection, timeline.session.session_pk)?; + session_titles::fill(connection, &mut timeline.session)?; Ok(timeline) } @@ -484,8 +483,7 @@ pub fn session_outline( events_by_turn.entry(event.turn_pk).or_default().push(event); } let mut timeline = assemble_timeline(session, turns, events_by_turn); - (timeline.session.title, timeline.session.purpose) = - session_titles::load(connection, timeline.session.session_pk)?; + session_titles::fill(connection, &mut timeline.session)?; Ok(timeline) } @@ -543,6 +541,7 @@ fn assemble_timeline( metadata: session.metadata, title: None, purpose: None, + parent_session_pk: None, }, turns: turn_timelines, } @@ -725,7 +724,7 @@ fn ingest_one_event( session_pk: session.id, turn_pk: turn.id, event_id: normalize_text(&event.event_id), - agent_name, + agent_name: agent_name.clone(), agent_version, session_id: normalize_text(&event.session_id), turn_index, @@ -771,8 +770,10 @@ fn ingest_one_event( session_pk: session.id, event_pk: usage_event_id, turn_index, + agent_name: &agent_name, event_type: event.event_type.as_str(), text: naming_text.as_deref(), + metadata: &event.metadata, observed_at, }, )?; diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index 5db09d0..9f43bfe 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -1,22 +1,40 @@ //! PostgreSQL persistence of session names read from title-generation calls. -use chrono::{DateTime, Utc}; +use std::collections::BTreeSet; + +use chrono::{DateTime, Duration, Utc}; use diesel::{ BoolExpressionMethods, Connection, ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl, TextExpressionMethods, + dsl::sql, + sql_types::{Nullable, Text}, }; +use serde_json::Value; use uuid::Uuid; use crate::{ - db::schema::{llm_usage_events, schema_backfills, session_titles}, + db::schema::{agent_sessions, llm_usage_events, schema_backfills, session_titles}, error::AppError, - usage::session_title::{ - TITLE_GENERATION_PURPOSE, TitleTask, may_carry_title, title_from_response, + usage::{ + SessionInfo, + session_title::{ + TITLE_GENERATION_PURPOSE, TitleTask, codex_naming_prompt, codex_turn_id, + codex_turn_id_range, may_carry_title, opens_with_prompt, title_from_response, + }, }, }; /// Marks the one-time scan of events stored before titles were captured. const BACKFILL_NAME: &str = "session_titles_v1"; +/// Marks the one-time matching of Codex naming threads captured before +/// matching existed. +const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; +/// Widest delivery delay tolerated between a naming thread and its task when +/// looking for candidates, in hours. +const LINK_DELIVERY_WINDOW_HOURS: i64 = 24; + +/// Stored title, purpose, and matched task session of one session. +type StoredTitle = (Option, Option, Option); /// One stored event considered for title capture. pub(super) struct CapturedEvent<'a> { @@ -24,8 +42,10 @@ pub(super) struct CapturedEvent<'a> { pub(super) session_pk: Uuid, pub(super) event_pk: Uuid, pub(super) turn_index: i32, + pub(super) agent_name: &'a str, pub(super) event_type: &'a str, pub(super) text: Option<&'a str>, + pub(super) metadata: &'a Value, pub(super) observed_at: DateTime, } @@ -34,6 +54,16 @@ struct NamingCall { task: TitleTask, /// Title with the response event and observation time that carried it. title: Option<(String, Uuid, DateTime)>, + /// Task message a Codex naming request quotes. + prompt: Option, +} + +/// A title to record for one session. +struct TitleUpdate<'a> { + session_pk: Uuid, + user_id: Uuid, + title: Option<(String, Uuid, DateTime)>, + purpose: Option<&'a str>, } /// Returns `true` when an event's text could belong to a naming call, so @@ -46,15 +76,21 @@ pub(super) fn is_candidate(event_type: &str, text: &str) -> bool { } } -/// Records the session title when the event completes a naming call. +/// Records the session title when the event completes a naming call, and +/// matches Codex naming threads with the task sessions they name. /// /// A naming request and its response arrive as separate events in either -/// order, so each side looks up the other within the same turn. +/// order, so each side looks up the other within the same turn. Likewise a +/// naming thread and its task session may arrive in either order, so both +/// sides try the match. pub(super) fn capture( connection: &mut PgConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { - naming_call(connection, event)?.map_or_else(|| Ok(()), |call| store(connection, event, call)) + match naming_call(connection, event)? { + Some(call) => store(connection, event, call), + None => match_pending_naming_threads(connection, event), + } } fn naming_call( @@ -64,13 +100,14 @@ fn naming_call( let Some(text) = event.text else { return Ok(None); }; + let session = event.session_pk; match event.event_type { "request" => { let Some(task) = TitleTask::from_request(text) else { return Ok(None); }; let response = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(event.session_pk)) + .filter(llm_usage_events::session_pk.eq(session)) .filter(llm_usage_events::turn_index.eq(event.turn_index)) .filter(llm_usage_events::event_type.eq("response")) .filter(llm_usage_events::observed_at.ge(event.observed_at)) @@ -87,14 +124,19 @@ fn naming_call( .and_then(title_from_response) .map(|title| (title, id, observed_at)) }); - Ok(Some(NamingCall { task, title })) + let prompt = codex_naming_prompt(text).map(str::to_owned); + Ok(Some(NamingCall { + task, + title, + prompt, + })) } "response" if may_carry_title(text) => { let Some(title) = title_from_response(text) else { return Ok(None); }; let request_text = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(event.session_pk)) + .filter(llm_usage_events::session_pk.eq(session)) .filter(llm_usage_events::turn_index.eq(event.turn_index)) .filter(llm_usage_events::event_type.eq("request")) .filter(llm_usage_events::observed_at.le(event.observed_at)) @@ -103,13 +145,13 @@ fn naming_call( .first::>(connection) .optional()? .flatten(); - Ok(request_text - .as_deref() - .and_then(TitleTask::from_request) - .map(|task| NamingCall { + Ok(request_text.as_deref().and_then(|request_text| { + TitleTask::from_request(request_text).map(|task| NamingCall { task, title: Some((title, event.event_pk, event.observed_at)), - })) + prompt: codex_naming_prompt(request_text).map(str::to_owned), + }) + })) } _ => Ok(None), } @@ -120,49 +162,77 @@ fn store( event: &CapturedEvent<'_>, call: NamingCall, ) -> Result<(), AppError> { + let session = event.session_pk; + let auxiliary = call.task == TitleTask::AuxiliarySession; + put_title( + connection, + TitleUpdate { + session_pk: session, + user_id: event.user_id, + title: call.title, + purpose: auxiliary.then_some(TITLE_GENERATION_PURPOSE), + }, + )?; + if !auxiliary { + return Ok(()); + } + let target = session_titles::table.find(session); + if let Some(turn_id) = codex_turn_id(event.metadata) { + diesel::update(target.filter(session_titles::naming_turn_id.is_null())) + .set(session_titles::naming_turn_id.eq(turn_id)) + .execute(connection)?; + } + if let Some(prompt) = call.prompt { + diesel::update(target.filter(session_titles::naming_prompt.is_null())) + .set(session_titles::naming_prompt.eq(prompt)) + .execute(connection)?; + } + match_naming_thread(connection, session) +} + +/// Inserts or updates a session title; a late or replayed naming call never +/// replaces a newer title. +fn put_title(connection: &mut PgConnection, update: TitleUpdate<'_>) -> Result<(), AppError> { let now = Utc::now(); - let purpose = - (call.task == TitleTask::AuxiliarySession).then(|| TITLE_GENERATION_PURPOSE.to_owned()); - let (title, title_event_pk, title_observed_at) = match call.title { + let (title, title_event_pk, title_observed_at) = match update.title { Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), None => (None, None, None), }; let existing = session_titles::table - .find(event.session_pk) + .find(update.session_pk) .select((session_titles::title_observed_at, session_titles::purpose)) .first::<(Option>, Option)>(connection) .optional()?; let Some((stored_at, stored_purpose)) = existing else { diesel::insert_into(session_titles::table) .values(( - session_titles::session_pk.eq(event.session_pk), - session_titles::user_id.eq(event.user_id), + session_titles::session_pk.eq(update.session_pk), + session_titles::user_id.eq(update.user_id), session_titles::title.eq(title), session_titles::title_event_pk.eq(title_event_pk), session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(purpose), + session_titles::purpose.eq(update.purpose), session_titles::updated_at.eq(now), )) .execute(connection)?; return Ok(()); }; - // A late or replayed naming call must not replace a newer title. let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); - let target = session_titles::table.find(event.session_pk); + let target = session_titles::table.find(update.session_pk); if newer { diesel::update(target) .set(( session_titles::title.eq(title), session_titles::title_event_pk.eq(title_event_pk), session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(purpose.or(stored_purpose)), + session_titles::purpose.eq(update.purpose.map(str::to_owned).or(stored_purpose)), session_titles::updated_at.eq(now), )) .execute(connection)?; - } else if stored_purpose.is_none() && purpose.is_some() { + } else if stored_purpose.is_none() && update.purpose.is_some() { diesel::update(target) .set(( - session_titles::purpose.eq(purpose), + session_titles::purpose.eq(update.purpose), session_titles::updated_at.eq(now), )) .execute(connection)?; @@ -170,54 +240,184 @@ fn store( Ok(()) } -/// Captures titles from events stored before capture existed, once. -pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { - connection.transaction(|transaction| { - let done = schema_backfills::table - .find(BACKFILL_NAME) - .select(schema_backfills::name) - .first::(transaction) - .optional()? - .is_some(); - if done { - return Ok(()); +/// Re-examines Codex naming threads whose turn started near this task +/// request's turn: an unmatched one may now have its task, and a matched one +/// that quotes this request too no longer has a single candidate. +fn match_pending_naming_threads( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + if event.event_type != "request" || event.agent_name != "codex" { + return Ok(()); + } + let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { + return Ok(()); + }; + let session = event.session_pk; + let nearby = session_titles::table + .filter(session_titles::user_id.eq(event.user_id)) + .filter(session_titles::title.is_not_null()) + .filter(session_titles::naming_turn_id.between(low, high)) + .select(( + session_titles::session_pk, + session_titles::naming_prompt, + session_titles::parent_session_pk, + session_titles::title_event_pk, + )) + .load::<(Uuid, Option, Option, Option)>(connection)?; + for (naming_session, prompt, parent, title_event_pk) in nearby { + if !prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { + continue; } - let requests = - llm_usage_events::table - .filter(llm_usage_events::event_type.eq("request")) - .filter( - llm_usage_events::text - .like("%You are naming a coding session%") - .or(llm_usage_events::text - .like("%Generate a concise, single-line task title%")), - ) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::user_id, - llm_usage_events::session_pk, - llm_usage_events::turn_index, - llm_usage_events::text, - llm_usage_events::observed_at, - )) - .load::<(Uuid, Uuid, Uuid, i32, Option, DateTime)>(transaction)?; - for (event_pk, user_id, session_pk, turn_index, text, observed_at) in requests { - capture( - transaction, - &CapturedEvent { - user_id, - session_pk, - event_pk, - turn_index, - event_type: "request", - text: text.as_deref(), - observed_at, - }, - )?; + match parent { + None => match_naming_thread(connection, naming_session)?, + Some(parent) if parent != session => { + unmatch(connection, naming_session, parent, title_event_pk)?; + } + Some(_) => {} } + } + Ok(()) +} + +/// Withdraws a match and the title it gave, because a second task session +/// quotes the same message. +fn unmatch( + connection: &mut PgConnection, + naming_session: Uuid, + task_session: Uuid, + title_event_pk: Option, +) -> Result<(), AppError> { + diesel::update(session_titles::table.find(naming_session)) + .set(session_titles::parent_session_pk.eq(None::)) + .execute(connection)?; + diesel::update( + session_titles::table + .find(task_session) + .filter(session_titles::title_event_pk.eq(title_event_pk)), + ) + .set(( + session_titles::title.eq(None::), + session_titles::title_event_pk.eq(None::), + session_titles::title_observed_at.eq(None::>), + session_titles::updated_at.eq(Utc::now()), + )) + .execute(connection)?; + Ok(()) +} + +/// Gives a Codex naming thread's title to the one task session whose turn +/// started near the naming turn with the quoted message. Zero or several +/// candidates leave the naming thread unmatched. +fn match_naming_thread( + connection: &mut PgConnection, + naming_session: Uuid, +) -> Result<(), AppError> { + let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt)) = + session_titles::table + .find(naming_session) + .filter(session_titles::parent_session_pk.is_null()) + .select(( + session_titles::user_id, + session_titles::title, + session_titles::title_event_pk, + session_titles::title_observed_at, + session_titles::naming_turn_id, + session_titles::naming_prompt, + )) + .first::<( + Uuid, + Option, + Option, + Option>, + Option, + Option, + )>(connection) + .optional()? + else { + return Ok(()); + }; + let (Some(title), Some(title_event_pk), Some(title_observed_at), Some(prompt)) = + (title, title_event_pk, title_observed_at, prompt) + else { + return Ok(()); + }; + let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { + return Ok(()); + }; + let window = Duration::hours(LINK_DELIVERY_WINDOW_HOURS); + let (Some(earliest), Some(latest)) = ( + title_observed_at.checked_sub_signed(window), + title_observed_at.checked_add_signed(window), + ) else { + return Ok(()); + }; + let device = agent_sessions::table + .find(naming_session) + .select(agent_sessions::device_context_id) + .first::(connection)?; + let candidates = llm_usage_events::table + .filter(llm_usage_events::user_id.eq(user_id)) + .filter(llm_usage_events::observed_at.between(earliest, latest)) + .filter(llm_usage_events::device_context_id.eq(device)) + .filter(llm_usage_events::agent_name.eq("codex")) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::session_pk.ne(naming_session)) + .filter( + sql::>("llm_usage_events.metadata ->> 'codex_turn_id'") + .between(low, high), + ) + .select((llm_usage_events::session_pk, llm_usage_events::metadata)) + .load::<(Uuid, Value)>(connection)?; + let sessions = candidates + .into_iter() + .filter(|(_, metadata)| opens_with_prompt(metadata, &prompt)) + .map(|(session, _)| session) + .collect::>(); + let mut sessions = sessions.into_iter(); + let (Some(task_session), None) = (sessions.next(), sessions.next()) else { + return Ok(()); + }; + diesel::update(session_titles::table.find(naming_session)) + .set(session_titles::parent_session_pk.eq(task_session)) + .execute(connection)?; + put_title( + connection, + TitleUpdate { + session_pk: task_session, + user_id, + title: Some((title, title_event_pk, title_observed_at)), + purpose: None, + }, + ) +} + +/// Runs the one-time scans that bring events stored before capture existed up +/// to date. +pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { + run_once(connection, BACKFILL_NAME, backfill_titles)?; + run_once(connection, LINK_BACKFILL_NAME, backfill_links) +} + +fn run_once( + connection: &mut PgConnection, + name: &str, + scan: fn(&mut PgConnection) -> Result<(), AppError>, +) -> Result<(), AppError> { + let done = schema_backfills::table + .find(name) + .select(schema_backfills::name) + .first::(connection) + .optional()? + .is_some(); + if done { + return Ok(()); + } + connection.transaction(|transaction| { + scan(transaction)?; diesel::insert_into(schema_backfills::table) .values(( - schema_backfills::name.eq(BACKFILL_NAME), + schema_backfills::name.eq(name), schema_backfills::completed_at.eq(Utc::now()), )) .execute(transaction)?; @@ -225,15 +425,96 @@ pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { }) } -/// Returns the stored title and purpose of one session. -pub(super) fn load( +/// Captures titles from naming requests stored before capture existed. +fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { + let requests = llm_usage_events::table + .filter(llm_usage_events::event_type.eq("request")) + .filter( + llm_usage_events::text + .like("%You are naming a coding session%") + .or(llm_usage_events::text.like("%Generate a concise, single-line task title%")), + ) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::session_pk, + llm_usage_events::turn_index, + llm_usage_events::text, + llm_usage_events::metadata, + llm_usage_events::observed_at, + )) + .load::<(Uuid, Uuid, Uuid, i32, Option, Value, DateTime)>(connection)?; + for (event_pk, user_id, session_pk, turn_index, text, metadata, observed_at) in requests { + capture( + connection, + &CapturedEvent { + user_id, + session_pk, + event_pk, + turn_index, + agent_name: "", + event_type: "request", + text: text.as_deref(), + metadata: &metadata, + observed_at, + }, + )?; + } + Ok(()) +} + +/// Matches naming threads whose titles were captured before matching existed. +fn backfill_links(connection: &mut PgConnection) -> Result<(), AppError> { + let naming_sessions = session_titles::table + .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) + .filter(session_titles::naming_turn_id.is_null()) + .select(session_titles::session_pk) + .load::(connection)?; + for naming_session in naming_sessions { + let request = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(naming_session)) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::text.like("%Generate a concise, single-line task title%")) + .order_by(llm_usage_events::observed_at.desc()) + .select((llm_usage_events::text, llm_usage_events::metadata)) + .first::<(Option, Value)>(connection) + .optional()?; + let Some((text, metadata)) = request else { + continue; + }; + diesel::update(session_titles::table.find(naming_session)) + .set(( + session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), + session_titles::naming_prompt.eq(text.as_deref().and_then(codex_naming_prompt)), + )) + .execute(connection)?; + match_naming_thread(connection, naming_session)?; + } + Ok(()) +} + +/// Adds the stored title, purpose, and matched task session to a session. +pub(super) fn fill( connection: &mut PgConnection, - session_pk: Uuid, -) -> Result<(Option, Option), AppError> { + session: &mut SessionInfo, +) -> Result<(), AppError> { + (session.title, session.purpose, session.parent_session_pk) = + load(connection, session.session_pk)?; + Ok(()) +} + +/// Returns the stored title and purpose of one session, and for a naming +/// thread the task session it was matched to. +fn load(connection: &mut PgConnection, session_pk: Uuid) -> Result { Ok(session_titles::table .find(session_pk) - .select((session_titles::title, session_titles::purpose)) - .first::<(Option, Option)>(connection) + .select(( + session_titles::title, + session_titles::purpose, + session_titles::parent_session_pk, + )) + .first::<(Option, Option, Option)>(connection) .optional()? - .unwrap_or_default()) + .unwrap_or((None, None, None))) } diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index 32ed061..ac7d060 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -147,7 +147,7 @@ fn ingest_one_event( session_pk: session_pk.to_string(), turn_pk: turn_pk.to_string(), event_id, - agent_name, + agent_name: agent_name.clone(), agent_version, session_id: normalize_text(&event.session_id), turn_index, @@ -182,8 +182,10 @@ fn ingest_one_event( session_pk, event_pk, turn_index, + agent_name: &agent_name, event_type: event.event_type.as_str(), text: text.as_deref(), + metadata: &event.metadata, observed_at, }, )?; @@ -674,8 +676,7 @@ pub(super) fn session_timeline( )); } let mut timeline = assemble_timeline(session, turns, events_by_turn); - (timeline.session.title, timeline.session.purpose) = - session_titles::load(connection, timeline.session.session_pk)?; + session_titles::fill(connection, &mut timeline.session)?; Ok(timeline) } @@ -806,8 +807,7 @@ pub(super) fn session_outline( events_by_turn.entry(event.turn_pk).or_default().push(event); } let mut timeline = assemble_timeline(session, turns, events_by_turn); - (timeline.session.title, timeline.session.purpose) = - session_titles::load(connection, timeline.session.session_pk)?; + session_titles::fill(connection, &mut timeline.session)?; Ok(timeline) } @@ -880,6 +880,7 @@ fn assemble_timeline( metadata: session.metadata, title: None, purpose: None, + parent_session_pk: None, }, turns: turn_timelines, } diff --git a/crates/abyss-backend/src/storage/sqlite/schema.rs b/crates/abyss-backend/src/storage/sqlite/schema.rs index af9182f..605aa01 100644 --- a/crates/abyss-backend/src/storage/sqlite/schema.rs +++ b/crates/abyss-backend/src/storage/sqlite/schema.rs @@ -144,6 +144,9 @@ diesel::table! { title_observed_at -> Nullable, purpose -> Nullable, updated_at -> BigInt, + naming_turn_id -> Nullable, + naming_prompt -> Nullable, + parent_session_pk -> Nullable, } } diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index 9939fb5..cb1e0c2 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -1,25 +1,43 @@ //! SQLite persistence of session names read from title-generation calls. +use std::collections::BTreeSet; + use chrono::Utc; use diesel::{ BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection, TextExpressionMethods, + dsl::sql, + sql_types::{Nullable, Text}, }; +use serde_json::Value; use uuid::Uuid; use super::{ models::parse_uuid, - schema::{llm_usage_events, schema_backfills, session_titles}, + schema::{agent_sessions, llm_usage_events, schema_backfills, session_titles}, }; use crate::{ error::AppError, - usage::session_title::{ - TITLE_GENERATION_PURPOSE, TitleTask, may_carry_title, title_from_response, + usage::{ + SessionInfo, + session_title::{ + TITLE_GENERATION_PURPOSE, TitleTask, codex_naming_prompt, codex_turn_id, + codex_turn_id_range, may_carry_title, opens_with_prompt, title_from_response, + }, }, }; /// Marks the one-time scan of events stored before titles were captured. const BACKFILL_NAME: &str = "session_titles_v1"; +/// Marks the one-time matching of Codex naming threads captured before +/// matching existed. +const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; +/// Widest delivery delay tolerated between a naming thread and its task when +/// looking for candidates, in microseconds. +const LINK_DELIVERY_WINDOW_MICROS: i64 = 24 * 60 * 60 * 1_000_000; + +/// Stored title, purpose, and matched task session of one session. +type StoredTitle = (Option, Option, Option); /// One stored event considered for title capture. pub(super) struct CapturedEvent<'a> { @@ -27,8 +45,10 @@ pub(super) struct CapturedEvent<'a> { pub(super) session_pk: Uuid, pub(super) event_pk: Uuid, pub(super) turn_index: i32, + pub(super) agent_name: &'a str, pub(super) event_type: &'a str, pub(super) text: Option<&'a str>, + pub(super) metadata: &'a Value, pub(super) observed_at: i64, } @@ -37,17 +57,33 @@ struct NamingCall { task: TitleTask, /// Title with the response event and observation time that carried it. title: Option<(String, String, i64)>, + /// Task message a Codex naming request quotes. + prompt: Option, +} + +/// A title to record for one session. +struct TitleUpdate<'a> { + session_pk: &'a str, + user_id: &'a str, + title: Option<(String, String, i64)>, + purpose: Option<&'a str>, } -/// Records the session title when the event completes a naming call. +/// Records the session title when the event completes a naming call, and +/// matches Codex naming threads with the task sessions they name. /// /// A naming request and its response arrive as separate events in either -/// order, so each side looks up the other within the same turn. +/// order, so each side looks up the other within the same turn. Likewise a +/// naming thread and its task session may arrive in either order, so both +/// sides try the match. pub(super) fn capture( connection: &mut SqliteConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { - naming_call(connection, event)?.map_or_else(|| Ok(()), |call| store(connection, event, call)) + match naming_call(connection, event)? { + Some(call) => store(connection, event, call), + None => match_pending_naming_threads(connection, event), + } } fn naming_call( @@ -81,7 +117,12 @@ fn naming_call( .and_then(title_from_response) .map(|title| (title, id, observed_at)) }); - Ok(Some(NamingCall { task, title })) + let prompt = codex_naming_prompt(text).map(str::to_owned); + Ok(Some(NamingCall { + task, + title, + prompt, + })) } "response" if may_carry_title(text) => { let Some(title) = title_from_response(text) else { @@ -97,13 +138,13 @@ fn naming_call( .first::>(connection) .optional()? .flatten(); - Ok(request_text - .as_deref() - .and_then(TitleTask::from_request) - .map(|task| NamingCall { + Ok(request_text.as_deref().and_then(|request_text| { + TitleTask::from_request(request_text).map(|task| NamingCall { task, title: Some((title, event.event_pk.to_string(), event.observed_at)), - })) + prompt: codex_naming_prompt(request_text).map(str::to_owned), + }) + })) } _ => Ok(None), } @@ -115,49 +156,76 @@ fn store( call: NamingCall, ) -> Result<(), AppError> { let session = event.session_pk.to_string(); + let auxiliary = call.task == TitleTask::AuxiliarySession; + put_title( + connection, + TitleUpdate { + session_pk: &session, + user_id: &event.user_id.to_string(), + title: call.title, + purpose: auxiliary.then_some(TITLE_GENERATION_PURPOSE), + }, + )?; + if !auxiliary { + return Ok(()); + } + let target = session_titles::table.find(&session); + if let Some(turn_id) = codex_turn_id(event.metadata) { + diesel::update(target.filter(session_titles::naming_turn_id.is_null())) + .set(session_titles::naming_turn_id.eq(turn_id)) + .execute(connection)?; + } + if let Some(prompt) = call.prompt { + diesel::update(target.filter(session_titles::naming_prompt.is_null())) + .set(session_titles::naming_prompt.eq(prompt)) + .execute(connection)?; + } + match_naming_thread(connection, &session) +} + +/// Inserts or updates a session title; a late or replayed naming call never +/// replaces a newer title. +fn put_title(connection: &mut SqliteConnection, update: TitleUpdate<'_>) -> Result<(), AppError> { let now = Utc::now().timestamp_micros(); - let purpose = - (call.task == TitleTask::AuxiliarySession).then(|| TITLE_GENERATION_PURPOSE.to_owned()); - let (title, title_event_pk, title_observed_at) = match call.title { + let (title, title_event_pk, title_observed_at) = match update.title { Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), None => (None, None, None), }; let existing = session_titles::table - .find(&session) + .find(update.session_pk) .select((session_titles::title_observed_at, session_titles::purpose)) .first::<(Option, Option)>(connection) .optional()?; let Some((stored_at, stored_purpose)) = existing else { diesel::insert_into(session_titles::table) .values(( - session_titles::session_pk.eq(&session), - session_titles::user_id.eq(event.user_id.to_string()), + session_titles::session_pk.eq(update.session_pk), + session_titles::user_id.eq(update.user_id), session_titles::title.eq(title), session_titles::title_event_pk.eq(title_event_pk), session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(purpose), + session_titles::purpose.eq(update.purpose), session_titles::updated_at.eq(now), )) .execute(connection)?; return Ok(()); }; - // A late or replayed naming call must not replace a newer title. let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); - let target = session_titles::table.find(&session); + let target = session_titles::table.find(update.session_pk); if newer { diesel::update(target) .set(( session_titles::title.eq(title), session_titles::title_event_pk.eq(title_event_pk), session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(purpose.or(stored_purpose)), + session_titles::purpose.eq(update.purpose.map(str::to_owned).or(stored_purpose)), session_titles::updated_at.eq(now), )) .execute(connection)?; - } else if stored_purpose.is_none() && purpose.is_some() { + } else if stored_purpose.is_none() && update.purpose.is_some() { diesel::update(target) .set(( - session_titles::purpose.eq(purpose), + session_titles::purpose.eq(update.purpose), session_titles::updated_at.eq(now), )) .execute(connection)?; @@ -165,10 +233,171 @@ fn store( Ok(()) } -/// Captures titles from events stored before capture existed, once. +/// Re-examines Codex naming threads whose turn started near this task +/// request's turn: an unmatched one may now have its task, and a matched one +/// that quotes this request too no longer has a single candidate. +fn match_pending_naming_threads( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + if event.event_type != "request" || event.agent_name != "codex" { + return Ok(()); + } + let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { + return Ok(()); + }; + let session = event.session_pk.to_string(); + let nearby = session_titles::table + .filter(session_titles::user_id.eq(event.user_id.to_string())) + .filter(session_titles::title.is_not_null()) + .filter(session_titles::naming_turn_id.between(low, high)) + .select(( + session_titles::session_pk, + session_titles::naming_prompt, + session_titles::parent_session_pk, + session_titles::title_event_pk, + )) + .load::<(String, Option, Option, Option)>(connection)?; + for (naming_session, prompt, parent, title_event_pk) in nearby { + if !prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { + continue; + } + match parent { + None => match_naming_thread(connection, &naming_session)?, + Some(parent) if parent != session => { + unmatch(connection, &naming_session, &parent, title_event_pk)?; + } + Some(_) => {} + } + } + Ok(()) +} + +/// Withdraws a match and the title it gave, because a second task session +/// quotes the same message. +fn unmatch( + connection: &mut SqliteConnection, + naming_session: &str, + task_session: &str, + title_event_pk: Option, +) -> Result<(), AppError> { + diesel::update(session_titles::table.find(naming_session)) + .set(session_titles::parent_session_pk.eq(None::)) + .execute(connection)?; + diesel::update( + session_titles::table + .find(task_session) + .filter(session_titles::title_event_pk.eq(title_event_pk)), + ) + .set(( + session_titles::title.eq(None::), + session_titles::title_event_pk.eq(None::), + session_titles::title_observed_at.eq(None::), + session_titles::updated_at.eq(Utc::now().timestamp_micros()), + )) + .execute(connection)?; + Ok(()) +} + +/// Gives a Codex naming thread's title to the one task session whose turn +/// started near the naming turn with the quoted message. Zero or several +/// candidates leave the naming thread unmatched. +fn match_naming_thread( + connection: &mut SqliteConnection, + naming_session: &str, +) -> Result<(), AppError> { + let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt)) = + session_titles::table + .find(naming_session) + .filter(session_titles::parent_session_pk.is_null()) + .select(( + session_titles::user_id, + session_titles::title, + session_titles::title_event_pk, + session_titles::title_observed_at, + session_titles::naming_turn_id, + session_titles::naming_prompt, + )) + .first::<( + String, + Option, + Option, + Option, + Option, + Option, + )>(connection) + .optional()? + else { + return Ok(()); + }; + let (Some(title), Some(title_event_pk), Some(title_observed_at), Some(prompt)) = + (title, title_event_pk, title_observed_at, prompt) + else { + return Ok(()); + }; + let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { + return Ok(()); + }; + let device = agent_sessions::table + .find(naming_session) + .select(agent_sessions::device_context_id) + .first::(connection)?; + let candidates = llm_usage_events::table + .filter(llm_usage_events::user_id.eq(&user_id)) + .filter(llm_usage_events::observed_at.between( + title_observed_at.saturating_sub(LINK_DELIVERY_WINDOW_MICROS), + title_observed_at.saturating_add(LINK_DELIVERY_WINDOW_MICROS), + )) + .filter(llm_usage_events::device_context_id.eq(&device)) + .filter(llm_usage_events::agent_name.eq("codex")) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::session_pk.ne(naming_session)) + .filter( + sql::>("json_extract(llm_usage_events.metadata, '$.codex_turn_id')") + .between(low, high), + ) + .select((llm_usage_events::session_pk, llm_usage_events::metadata)) + .load::<(String, String)>(connection)?; + let sessions = candidates + .into_iter() + .filter(|(_, metadata)| { + serde_json::from_str::(metadata) + .is_ok_and(|metadata| opens_with_prompt(&metadata, &prompt)) + }) + .map(|(session, _)| session) + .collect::>(); + let mut sessions = sessions.into_iter(); + let (Some(task_session), None) = (sessions.next(), sessions.next()) else { + return Ok(()); + }; + diesel::update(session_titles::table.find(naming_session)) + .set(session_titles::parent_session_pk.eq(&task_session)) + .execute(connection)?; + put_title( + connection, + TitleUpdate { + session_pk: &task_session, + user_id: &user_id, + title: Some((title, title_event_pk, title_observed_at)), + purpose: None, + }, + ) +} + +/// Runs the one-time scans that bring events stored before capture existed up +/// to date. pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { + run_once(connection, BACKFILL_NAME, backfill_titles)?; + run_once(connection, LINK_BACKFILL_NAME, backfill_links) +} + +fn run_once( + connection: &mut SqliteConnection, + name: &str, + scan: fn(&mut SqliteConnection) -> Result<(), AppError>, +) -> Result<(), AppError> { let done = schema_backfills::table - .find(BACKFILL_NAME) + .find(name) .select(schema_backfills::name) .first::(connection) .optional()? @@ -177,42 +406,10 @@ pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError return Ok(()); } connection.immediate_transaction(|transaction| { - let requests = - llm_usage_events::table - .filter(llm_usage_events::event_type.eq("request")) - .filter( - llm_usage_events::text - .like("%You are naming a coding session%") - .or(llm_usage_events::text - .like("%Generate a concise, single-line task title%")), - ) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::user_id, - llm_usage_events::session_pk, - llm_usage_events::turn_index, - llm_usage_events::text, - llm_usage_events::observed_at, - )) - .load::<(String, String, String, i32, Option, i64)>(transaction)?; - for (event_pk, user_id, session_pk, turn_index, text, observed_at) in requests { - capture( - transaction, - &CapturedEvent { - user_id: parse_uuid(&user_id, "event user id")?, - session_pk: parse_uuid(&session_pk, "event session id")?, - event_pk: parse_uuid(&event_pk, "event id")?, - turn_index, - event_type: "request", - text: text.as_deref(), - observed_at, - }, - )?; - } + scan(transaction)?; diesel::insert_into(schema_backfills::table) .values(( - schema_backfills::name.eq(BACKFILL_NAME), + schema_backfills::name.eq(name), schema_backfills::completed_at.eq(Utc::now().timestamp_micros()), )) .execute(transaction)?; @@ -220,19 +417,107 @@ pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError }) } -/// Returns the stored title and purpose of one session. -pub(super) fn load( +/// Captures titles from naming requests stored before capture existed. +fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { + let requests = llm_usage_events::table + .filter(llm_usage_events::event_type.eq("request")) + .filter( + llm_usage_events::text + .like("%You are naming a coding session%") + .or(llm_usage_events::text.like("%Generate a concise, single-line task title%")), + ) + .order_by(llm_usage_events::observed_at.asc()) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::session_pk, + llm_usage_events::turn_index, + llm_usage_events::text, + llm_usage_events::metadata, + llm_usage_events::observed_at, + )) + .load::<(String, String, String, i32, Option, String, i64)>(connection)?; + for (event_pk, user_id, session_pk, turn_index, text, metadata, observed_at) in requests { + let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); + capture( + connection, + &CapturedEvent { + user_id: parse_uuid(&user_id, "event user id")?, + session_pk: parse_uuid(&session_pk, "event session id")?, + event_pk: parse_uuid(&event_pk, "event id")?, + turn_index, + agent_name: "", + event_type: "request", + text: text.as_deref(), + metadata: &metadata, + observed_at, + }, + )?; + } + Ok(()) +} + +/// Matches naming threads whose titles were captured before matching existed. +fn backfill_links(connection: &mut SqliteConnection) -> Result<(), AppError> { + let naming_sessions = session_titles::table + .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) + .filter(session_titles::naming_turn_id.is_null()) + .select(session_titles::session_pk) + .load::(connection)?; + for naming_session in naming_sessions { + let request = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(&naming_session)) + .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::text.like("%Generate a concise, single-line task title%")) + .order_by(llm_usage_events::observed_at.desc()) + .select((llm_usage_events::text, llm_usage_events::metadata)) + .first::<(Option, String)>(connection) + .optional()?; + let Some((text, metadata)) = request else { + continue; + }; + let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); + diesel::update(session_titles::table.find(&naming_session)) + .set(( + session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), + session_titles::naming_prompt.eq(text.as_deref().and_then(codex_naming_prompt)), + )) + .execute(connection)?; + match_naming_thread(connection, &naming_session)?; + } + Ok(()) +} + +/// Adds the stored title, purpose, and matched task session to a session. +pub(super) fn fill( connection: &mut SqliteConnection, - session_pk: Uuid, -) -> Result<(Option, Option), AppError> { - Ok(session_titles::table + session: &mut SessionInfo, +) -> Result<(), AppError> { + (session.title, session.purpose, session.parent_session_pk) = + load(connection, session.session_pk)?; + Ok(()) +} + +/// Returns the stored title and purpose of one session, and for a naming +/// thread the task session it was matched to. +fn load(connection: &mut SqliteConnection, session_pk: Uuid) -> Result { + let Some((title, purpose, parent)) = session_titles::table .find(session_pk.to_string()) - .select((session_titles::title, session_titles::purpose)) - .first::<(Option, Option)>(connection) + .select(( + session_titles::title, + session_titles::purpose, + session_titles::parent_session_pk, + )) + .first::<(Option, Option, Option)>(connection) .optional()? - .unwrap_or_default()) + else { + return Ok((None, None, None)); + }; + let parent = parent + .map(|parent| parse_uuid(&parent, "parent session id")) + .transpose()?; + Ok((title, purpose, parent)) } - #[cfg(test)] mod tests { use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; @@ -290,7 +575,70 @@ mod tests { .select(agent_sessions::id) .first::(database) .expect("session should exist"); - load(database, Uuid::parse_str(&pk).expect("uuid")).expect("title should load") + let (title, purpose, _) = + load(database, Uuid::parse_str(&pk).expect("uuid")).expect("title should load"); + (title, purpose) + } + + fn session_pk(database: &mut SqliteConnection, session: &str) -> Uuid { + let pk = agent_sessions::table + .filter(agent_sessions::session_id.eq(session)) + .select(agent_sessions::id) + .first::(database) + .expect("session should exist"); + Uuid::parse_str(&pk).expect("uuid") + } + + fn parent_of(database: &mut SqliteConnection, session: &str) -> Option { + let pk = session_pk(database, session); + load(database, pk).expect("title should load").2 + } + + const NAMING_TURN: &str = "01a0f320-138e-7d21-99ea-03a7ae2a9b50"; + const TASK_TURN: &str = "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"; + const LATE_TURN: &str = "01a0f330-1350-71c2-9c93-5fc5fd7a6bfb"; + const TASK: &str = "画一只鹈鹕骑自行车，不要联网"; + + fn codex_naming_thread(session: &str) -> [Value; 2] { + let mut request = event( + "codex", + session, + &format!("{session}-r"), + "2026-09-30T00:00:01Z", + "request", + &format!("{CODEX_NAMING}\n\nUser prompt:\n画一只鹈鹕骑自行车"), + ); + request["metadata"] = json!({"codex_turn_id": NAMING_TURN}); + let mut response = event( + "codex", + session, + &format!("{session}-s"), + "2026-09-30T00:00:02Z", + "response", + r#"{"title":"Draw a pelican"}"#, + ); + response["metadata"] = json!({"codex_turn_id": NAMING_TURN}); + [request, response] + } + + fn codex_task(session: &str, turn_id: &str, message: &str) -> Value { + let mut request = event( + "codex", + session, + &format!("{session}-{turn_id}"), + "2026-09-30T00:03:00Z", + "request", + message, + ); + request["metadata"] = json!({ + "codex_turn_id": turn_id, + "conversation": {"messages": [ + {"role": "developer", "text": "You are Codex"}, + {"role": "user", "text": "…"}, + {"role": "user", "text": message}, + ]}, + }); + request } #[test] @@ -482,7 +830,57 @@ mod tests { .count() .get_result::(&mut database) .expect("count"), - 1 + 2, + "titles and Codex links are each scanned once" + ); + } + + #[test] + fn names_the_codex_task_its_naming_thread_quotes_in_either_order() { + for naming_first in [true, false] { + let mut database = database(); + let naming = codex_naming_thread("title-thread"); + let task = codex_task("task", TASK_TURN, TASK); + if naming_first { + ingest(&mut database, &naming); + ingest(&mut database, &[task]); + } else { + ingest(&mut database, &[task]); + ingest(&mut database, &naming); + } + + assert_eq!( + session_title(&mut database, "task"), + (Some("Draw a pelican".to_owned()), None) + ); + let task_pk = session_pk(&mut database, "task"); + assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); + } + } + + #[test] + fn leaves_codex_naming_threads_unmatched_without_one_clear_task() { + let mut database = database(); + ingest(&mut database, &codex_naming_thread("title-thread")); + ingest( + &mut database, + &[ + codex_task("other-message", TASK_TURN, "写一个排序函数"), + codex_task("too-late", LATE_TURN, TASK), + ], + ); + assert_eq!(parent_of(&mut database, "title-thread"), None); + assert_eq!(session_title(&mut database, "too-late"), (None, None)); + + // The same message sent twice at once cannot be told apart. + ingest( + &mut database, + &[ + codex_task("first", TASK_TURN, TASK), + codex_task("second", TASK_TURN, TASK), + ], ); + assert_eq!(parent_of(&mut database, "title-thread"), None); + assert_eq!(session_title(&mut database, "second"), (None, None)); } } diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index ddec30d..1c70ee4 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -535,6 +535,8 @@ pub struct SessionInfo { /// Why the session exists when it only serves the Agent, such as /// `title_generation`; `None` for ordinary sessions. pub purpose: Option, + /// For a naming thread, the task session its title was matched to. + pub parent_session_pk: Option, } /// Extracts the endpoint working directory retained in session metadata. diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index 55d1b8c..7474b2d 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -5,8 +5,16 @@ //! the name can be read back from them without guessing: a call counts only //! when its request carries an official naming instruction and its response //! consists solely of title objects that agree. +//! +//! Codex asks from a separate thread whose requests carry no link to the task +//! they name. The naming request quotes the task's user message, and Codex +//! starts the naming turn right after the task turn, so a task turn is matched +//! when it starts close to the naming turn and its user message opens with the +//! quoted prompt. use serde::Deserialize; +use serde_json::Value; +use uuid::Uuid; /// Claude Code instruction for naming the session it runs in. const CLAUDE_CODE_NAMING_INSTRUCTION: &str = "You are naming a coding session"; @@ -15,6 +23,12 @@ const CODEX_NAMING_INSTRUCTION: &str = "Generate a concise, single-line task tit /// Longest title kept, in characters; longer answers are not titles. const MAX_TITLE_CHARS: usize = 200; +/// Precedes the quoted task message in a Codex naming request. +const CODEX_PROMPT_MARKER: &str = "User prompt:\n"; +/// Widest start gap between a Codex naming turn and the task turn it names. +/// Observed gaps are mostly under half a second and at most about 31 s. +const CODEX_LINK_WINDOW_MS: u64 = 60_000; + /// Session purpose stored for Codex's standalone naming threads. pub const TITLE_GENERATION_PURPOSE: &str = "title_generation"; @@ -74,6 +88,55 @@ pub fn may_carry_title(text: &str) -> bool { trimmed.starts_with('{') && trimmed.contains("\"title\"") } +/// Returns the task message a Codex naming request quotes. Codex may cut long +/// messages short, so the quote can be a prefix of the task message. +pub fn codex_naming_prompt(request_text: &str) -> Option<&str> { + let (_, prompt) = request_text.rsplit_once(CODEX_PROMPT_MARKER)?; + Some(prompt.trim()).filter(|prompt| !prompt.is_empty()) +} + +/// Returns the Codex turn ID an event's collector recorded. +pub fn codex_turn_id(metadata: &Value) -> Option<&str> { + metadata.get("codex_turn_id")?.as_str() +} + +/// Returns the range of Codex turn IDs that start within the link window of +/// `turn_id`. Codex turn IDs are version 7 UUIDs, whose text sorts by start +/// time held in the top 48 bits. +pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { + const TIME_SHIFT: u32 = 80; + let uuid = Uuid::parse_str(turn_id).ok()?; + if uuid.get_version_num() != 7 { + return None; + } + let started_ms = u64::try_from(uuid.as_u128().checked_shr(TIME_SHIFT)?).ok()?; + let low = + u128::from(started_ms.saturating_sub(CODEX_LINK_WINDOW_MS)).checked_shl(TIME_SHIFT)?; + let high = u128::from(started_ms.saturating_add(CODEX_LINK_WINDOW_MS)) + .checked_shl(TIME_SHIFT)? + | u128::MAX.checked_shr(u128::BITS - TIME_SHIFT)?; + Some(( + Uuid::from_u128(low).to_string(), + Uuid::from_u128(high).to_string(), + )) +} + +/// Returns `true` when a request's latest user message is the task a naming +/// request quoted. +pub fn opens_with_prompt(metadata: &Value, prompt: &str) -> bool { + metadata + .pointer("/conversation/messages") + .and_then(Value::as_array) + .and_then(|messages| { + messages + .iter() + .rev() + .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) + }) + .and_then(|message| message.get("text")?.as_str()) + .is_some_and(|text| text.trim_start().starts_with(prompt)) +} + #[cfg(test)] mod tests { use super::*; @@ -122,6 +185,42 @@ mod tests { ); } + #[test] + fn reads_the_quoted_task_from_codex_naming_requests() { + assert_eq!( + codex_naming_prompt("Generate a concise title.\n\nUser prompt:\n画一只鹈鹕\n"), + Some("画一只鹈鹕") + ); + assert_eq!(codex_naming_prompt("User prompt:\n "), None); + assert_eq!(codex_naming_prompt("no quote"), None); + } + + #[test] + fn bounds_codex_turn_ids_by_start_time() { + let (low, high) = + codex_turn_id_range("01a0f320-138e-7d21-99ea-03a7ae2a9b50").expect("UUIDv7"); + assert!(low.as_str() < "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"); + assert!(high.as_str() > "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"); + assert!(high.as_str() < "01a0f321-0000-7000-8000-000000000000"); + assert_eq!(codex_turn_id_range("turn-1"), None); + assert_eq!( + codex_turn_id_range("f81d4fae-7dec-11d0-a765-00a0c91e6bf6"), + None + ); + } + + #[test] + fn matches_the_latest_user_message_by_prefix() { + let metadata = serde_json::json!({"conversation": {"messages": [ + {"role": "user", "text": "…"}, + {"role": "user", "text": "画一只鹈鹕骑自行车，不要联网"}, + {"role": "assistant", "text": "好的"}, + ]}}); + assert!(opens_with_prompt(&metadata, "画一只鹈鹕")); + assert!(!opens_with_prompt(&metadata, "")); + assert!(!opens_with_prompt(&serde_json::json!({}), "画")); + } + #[test] fn prefilters_candidate_responses() { assert!(may_carry_title(" {\"title\": \"x\"}")); diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 05af664..5ebe322 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -745,6 +745,7 @@ mod tests { metadata: json!({}), title: None, purpose: None, + parent_session_pk: None, }, turns: vec![TurnTimeline { turn_pk: Uuid::nil(), From de51e225291ec09d6e1c3a7a69e33072afbbd737 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Thu, 1 Oct 2026 21:10:32 -0700 Subject: [PATCH 12/30] refactor(trajectory): reuse parsed messages when assigning streams --- crates/abyss-backend/src/usage/trajectory.rs | 14 ++--- .../src/usage/trajectory/raw_body.rs | 52 ++++++++----------- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 64da961..442f879 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -177,12 +177,10 @@ impl Merger { fn request(&mut self, event: &UsageEventResponse) { self.pending_compaction = None; let first_new = self.items.len(); - if let Some(context) = RawProjection::context(event) { - let key = hash_str(&context); - let next = self.streams.len(); - self.current_stream = *self.streams.entry(key).or_insert(next); - } - if let Some(items) = self.raw.project(event, self.current_stream) { + if let Some(items) = self + .raw + .project(event, &mut self.current_stream, &mut self.streams) + { self.items.extend(items); } else { self.unclassified_request(event); @@ -293,7 +291,9 @@ impl Merger { fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; - let projected = self.raw.project(event, stream); + let projected = self + .raw + .project(event, &mut self.current_stream, &mut self.streams); if let Some(index) = self.pending_compaction.take() { // The response to the compaction prompt is the handoff summary. if let Some(summary) = event.text.as_deref().map(str::trim) diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs index 35f80f8..27cb03c 100644 --- a/crates/abyss-backend/src/usage/trajectory/raw_body.rs +++ b/crates/abyss-backend/src/usage/trajectory/raw_body.rs @@ -46,42 +46,34 @@ struct StreamHistory { } impl RawProjection { - pub(super) fn context(event: &UsageEventResponse) -> Option { - let values = bodies(event)?; - let payload = request_payload(&values)?; - if payload - .get("previous_response_id") - .is_some_and(Value::is_string) - && payload.get("instructions").is_none() - && payload.get("system").is_none() - { - return None; - } - Some( - request_messages(payload) - .into_iter() - .filter(|m| m.kind() == TrajectoryItemKind::Context) - .map(|m| format!("{}: {}", m.role, m.text)) - .collect::>() - .join("\n\n"), - ) - } - pub(super) fn project( &mut self, event: &UsageEventResponse, - stream: usize, + stream: &mut usize, + stream_ids: &mut HashMap, ) -> Option> { let values = bodies(event)?; let request = event.event_type == "request"; let (messages, incremental) = if request { let payload = request_payload(&values)?; - ( - request_messages(payload), - payload - .get("previous_response_id") - .is_some_and(Value::is_string), - ) + let messages = request_messages(payload); + let incremental = payload + .get("previous_response_id") + .is_some_and(Value::is_string); + if !incremental + || payload.get("instructions").is_some() + || payload.get("system").is_some() + { + let context = messages + .iter() + .filter(|message| message.kind() == TrajectoryItemKind::Context) + .map(|message| format!("{}: {}", message.role, message.text)) + .collect::>() + .join("\n\n"); + let next = stream_ids.len(); + *stream = *stream_ids.entry(super::hash_str(&context)).or_insert(next); + } + (messages, incremental) } else { let mut messages = response_messages(&values); if messages.is_empty() @@ -100,7 +92,7 @@ impl RawProjection { if !request && messages.is_empty() { return None; } - let state = self.streams.entry(stream).or_default(); + let state = self.streams.entry(*stream).or_default(); let mut cursor = 0; let mut next = Vec::new(); let mut added = Vec::new(); @@ -133,7 +125,7 @@ impl RawProjection { } } if !known && !message.text.is_empty() { - let mut item = TrajectoryItem::new(message.kind(), stream, event); + let mut item = TrajectoryItem::new(message.kind(), *stream, event); item.text = Some(message.text); added.push(item); } From 0fb989131f2c28826920a1186113166d756ba22e Mon Sep 17 00:00:00 2001 From: Auceptin Date: Thu, 1 Oct 2026 23:50:45 -0700 Subject: [PATCH 13/30] fix: scope session title adapters and recover Codex task links --- .gitignore | 1 + README.md | 8 + .../src/storage/postgres/repository.rs | 6 +- .../src/storage/postgres/session_titles.rs | 162 +++++---- .../src/storage/sqlite/session_titles.rs | 322 ++++++++++++++--- crates/abyss-backend/src/usage/mod.rs | 1 + .../abyss-backend/src/usage/provider_body.rs | 134 +++++++ .../abyss-backend/src/usage/session_title.rs | 337 ++++++++++++++++-- .../src/usage/trajectory/raw_body.rs | 136 +------ scripts/tests/blackbox_api.py | 3 + scripts/tests/session_titles_api.py | 83 +++++ 11 files changed, 914 insertions(+), 279 deletions(-) create mode 100644 crates/abyss-backend/src/usage/provider_body.rs create mode 100644 scripts/tests/session_titles_api.py diff --git a/.gitignore b/.gitignore index c901f2f..9847753 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target/ +__pycache__/ .DS_Store .env diff --git a/README.md b/README.md index 0e37e07..f44a85a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,14 @@ and optional full-text search. - Health and storage-readiness probes. - Embedded migrations for both supported storage profiles. +Session titles have dedicated adapters for Codex and Claude Code. Claude Code +names its own session; Codex names a separate task thread, matched by quoted +user content, turn start time, device, and available installation identity. +Only a unique task candidate receives the generated title. Other Agents keep +the default session identity and are not classified using these Agents' naming +templates. Both adapters read archived protocol messages and support older +structured event records. Auxiliary title calls retain their own events and usage. + Exactly one storage profile is compiled into a binary: - `postgres-es` uses PostgreSQL as the source of truth and optionally projects diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index a378a9b..bd0e145 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -708,10 +708,6 @@ fn ingest_one_event( )?; let tokens = normalized_tokens(event)?; let text = normalized_optional(event.text.as_deref()); - let naming_text = text - .as_deref() - .filter(|text| session_titles::is_candidate(event.event_type.as_str(), text)) - .map(str::to_owned); // `event_id` is the idempotency key supplied by the collector. Duplicate // observations are accepted as no-ops instead of failing the whole batch. @@ -772,7 +768,7 @@ fn ingest_one_event( turn_index, agent_name: &agent_name, event_type: event.event_type.as_str(), - text: naming_text.as_deref(), + text: event.text.as_deref(), metadata: &event.metadata, observed_at, }, diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index 9f43bfe..78cea15 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -18,8 +18,8 @@ use crate::{ usage::{ SessionInfo, session_title::{ - TITLE_GENERATION_PURPOSE, TitleTask, codex_naming_prompt, codex_turn_id, - codex_turn_id_range, may_carry_title, opens_with_prompt, title_from_response, + CodexRequest, NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, + codex_turn_id_range, may_carry_title, title_from_response, }, }, }; @@ -29,6 +29,8 @@ const BACKFILL_NAME: &str = "session_titles_v1"; /// Marks the one-time matching of Codex naming threads captured before /// matching existed. const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; +/// Rebuild links after switching to archived request bodies and scoped adapters. +const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; /// Widest delivery delay tolerated between a naming thread and its task when /// looking for candidates, in hours. const LINK_DELIVERY_WINDOW_HOURS: i64 = 24; @@ -66,16 +68,6 @@ struct TitleUpdate<'a> { purpose: Option<&'a str>, } -/// Returns `true` when an event's text could belong to a naming call, so -/// ingestion copies text only for candidates. -pub(super) fn is_candidate(event_type: &str, text: &str) -> bool { - match event_type { - "request" => TitleTask::from_request(text).is_some(), - "response" => may_carry_title(text), - _ => false, - } -} - /// Records the session title when the event completes a naming call, and /// matches Codex naming threads with the task sessions they name. /// @@ -97,13 +89,11 @@ fn naming_call( connection: &mut PgConnection, event: &CapturedEvent<'_>, ) -> Result, AppError> { - let Some(text) = event.text else { - return Ok(None); - }; let session = event.session_pk; match event.event_type { "request" => { - let Some(task) = TitleTask::from_request(text) else { + let Some(naming) = NamingRequest::parse(event.agent_name, event.text, event.metadata) + else { return Ok(None); }; let response = llm_usage_events::table @@ -124,32 +114,32 @@ fn naming_call( .and_then(title_from_response) .map(|title| (title, id, observed_at)) }); - let prompt = codex_naming_prompt(text).map(str::to_owned); Ok(Some(NamingCall { - task, + task: naming.task, title, - prompt, + prompt: naming.prompt, })) } - "response" if may_carry_title(text) => { - let Some(title) = title_from_response(text) else { + "response" if event.text.is_some_and(may_carry_title) => { + let Some(title) = event.text.and_then(title_from_response) else { return Ok(None); }; - let request_text = llm_usage_events::table + let request = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(session)) .filter(llm_usage_events::turn_index.eq(event.turn_index)) .filter(llm_usage_events::event_type.eq("request")) .filter(llm_usage_events::observed_at.le(event.observed_at)) .order_by(llm_usage_events::observed_at.desc()) - .select(llm_usage_events::text) - .first::>(connection) - .optional()? - .flatten(); - Ok(request_text.as_deref().and_then(|request_text| { - TitleTask::from_request(request_text).map(|task| NamingCall { - task, - title: Some((title, event.event_pk, event.observed_at)), - prompt: codex_naming_prompt(request_text).map(str::to_owned), + .select((llm_usage_events::text, llm_usage_events::metadata)) + .first::<(Option, Value)>(connection) + .optional()?; + Ok(request.and_then(|(text, metadata)| { + NamingRequest::parse(event.agent_name, text.as_deref(), &metadata).map(|naming| { + NamingCall { + task: naming.task, + title: Some((title, event.event_pk, event.observed_at)), + prompt: naming.prompt, + } }) })) } @@ -253,28 +243,29 @@ fn match_pending_naming_threads( let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { return Ok(()); }; - let session = event.session_pk; let nearby = session_titles::table .filter(session_titles::user_id.eq(event.user_id)) + .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) .filter(session_titles::title.is_not_null()) + .filter( + session_titles::parent_session_pk + .is_null() + .or(session_titles::parent_session_pk.ne(event.session_pk)), + ) .filter(session_titles::naming_turn_id.between(low, high)) - .select(( - session_titles::session_pk, - session_titles::naming_prompt, - session_titles::parent_session_pk, - session_titles::title_event_pk, - )) - .load::<(Uuid, Option, Option, Option)>(connection)?; - for (naming_session, prompt, parent, title_event_pk) in nearby { - if !prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { - continue; - } - match parent { - None => match_naming_thread(connection, naming_session)?, - Some(parent) if parent != session => { - unmatch(connection, naming_session, parent, title_event_pk)?; - } - Some(_) => {} + .select((session_titles::session_pk, session_titles::naming_prompt)) + .load::<(Uuid, Option)>(connection)?; + if nearby.is_empty() { + return Ok(()); + } + let Some(request) = CodexRequest::parse(event.metadata) else { + return Ok(()); + }; + for (naming_session, prompt) in nearby { + if prompt.is_some_and(|prompt| request.opens_with_prompt(&prompt)) { + // Recompute the complete candidate set before adding or withdrawing + // a link. A different device or installation is not a conflict. + match_naming_thread(connection, naming_session)?; } } Ok(()) @@ -313,10 +304,9 @@ fn match_naming_thread( connection: &mut PgConnection, naming_session: Uuid, ) -> Result<(), AppError> { - let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt)) = + let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt, old_parent)) = session_titles::table .find(naming_session) - .filter(session_titles::parent_session_pk.is_null()) .select(( session_titles::user_id, session_titles::title, @@ -324,6 +314,7 @@ fn match_naming_thread( session_titles::title_observed_at, session_titles::naming_turn_id, session_titles::naming_prompt, + session_titles::parent_session_pk, )) .first::<( Uuid, @@ -332,6 +323,7 @@ fn match_naming_thread( Option>, Option, Option, + Option, )>(connection) .optional()? else { @@ -352,6 +344,9 @@ fn match_naming_thread( ) else { return Ok(()); }; + let Some(naming) = naming_context(connection, naming_session, turn_id.as_deref())? else { + return Ok(()); + }; let device = agent_sessions::table .find(naming_session) .select(agent_sessions::device_context_id) @@ -371,11 +366,22 @@ fn match_naming_thread( .load::<(Uuid, Value)>(connection)?; let sessions = candidates .into_iter() - .filter(|(_, metadata)| opens_with_prompt(metadata, &prompt)) + .filter(|(_, metadata)| { + CodexRequest::parse(metadata).is_some_and(|request| request.matches(&naming, &prompt)) + }) .map(|(session, _)| session) .collect::>(); let mut sessions = sessions.into_iter(); - let (Some(task_session), None) = (sessions.next(), sessions.next()) else { + let task_session = match (sessions.next(), sessions.next()) { + (Some(session), None) => Some(session), + _ => None, + }; + if let Some(parent) = old_parent + && Some(parent) != task_session + { + unmatch(connection, naming_session, parent, Some(title_event_pk))?; + } + let Some(task_session) = task_session else { return Ok(()); }; diesel::update(session_titles::table.find(naming_session)) @@ -392,11 +398,32 @@ fn match_naming_thread( ) } +/// Reads the naming request's protocol evidence without changing its archive. +fn naming_context( + connection: &mut PgConnection, + naming_session: Uuid, + turn_id: Option<&str>, +) -> Result, AppError> { + let metadata = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(naming_session)) + .filter(llm_usage_events::event_type.eq("request")) + .filter(sql::>("llm_usage_events.metadata ->> 'codex_turn_id'").eq(turn_id)) + .order_by(llm_usage_events::observed_at.desc()) + .select(llm_usage_events::metadata) + .first::(connection) + .optional()?; + let Some(metadata) = metadata else { + return Ok(None); + }; + Ok(CodexRequest::parse(&metadata)) +} + /// Runs the one-time scans that bring events stored before capture existed up /// to date. pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { run_once(connection, BACKFILL_NAME, backfill_titles)?; - run_once(connection, LINK_BACKFILL_NAME, backfill_links) + run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; + run_once(connection, LINK_REPAIR_NAME, backfill_links) } fn run_once( @@ -429,6 +456,7 @@ fn run_once( fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { let requests = llm_usage_events::table .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::agent_name.eq_any(["codex", "claude-code"])) .filter( llm_usage_events::text .like("%You are naming a coding session%") @@ -440,12 +468,24 @@ fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { llm_usage_events::user_id, llm_usage_events::session_pk, llm_usage_events::turn_index, + llm_usage_events::agent_name, llm_usage_events::text, llm_usage_events::metadata, llm_usage_events::observed_at, )) - .load::<(Uuid, Uuid, Uuid, i32, Option, Value, DateTime)>(connection)?; - for (event_pk, user_id, session_pk, turn_index, text, metadata, observed_at) in requests { + .load::<( + Uuid, + Uuid, + Uuid, + i32, + String, + Option, + Value, + DateTime, + )>(connection)?; + for (event_pk, user_id, session_pk, turn_index, agent_name, text, metadata, observed_at) in + requests + { capture( connection, &CapturedEvent { @@ -453,7 +493,7 @@ fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { session_pk, event_pk, turn_index, - agent_name: "", + agent_name: &agent_name, event_type: "request", text: text.as_deref(), metadata: &metadata, @@ -468,14 +508,13 @@ fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { fn backfill_links(connection: &mut PgConnection) -> Result<(), AppError> { let naming_sessions = session_titles::table .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .filter(session_titles::naming_turn_id.is_null()) .select(session_titles::session_pk) .load::(connection)?; for naming_session in naming_sessions { let request = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(naming_session)) .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::text.like("%Generate a concise, single-line task title%")) + .filter(llm_usage_events::agent_name.eq("codex")) .order_by(llm_usage_events::observed_at.desc()) .select((llm_usage_events::text, llm_usage_events::metadata)) .first::<(Option, Value)>(connection) @@ -483,10 +522,13 @@ fn backfill_links(connection: &mut PgConnection) -> Result<(), AppError> { let Some((text, metadata)) = request else { continue; }; + let Some(naming) = NamingRequest::parse("codex", text.as_deref(), &metadata) else { + continue; + }; diesel::update(session_titles::table.find(naming_session)) .set(( session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), - session_titles::naming_prompt.eq(text.as_deref().and_then(codex_naming_prompt)), + session_titles::naming_prompt.eq(naming.prompt), )) .execute(connection)?; match_naming_thread(connection, naming_session)?; diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index cb1e0c2..b303f9c 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -21,8 +21,8 @@ use crate::{ usage::{ SessionInfo, session_title::{ - TITLE_GENERATION_PURPOSE, TitleTask, codex_naming_prompt, codex_turn_id, - codex_turn_id_range, may_carry_title, opens_with_prompt, title_from_response, + CodexRequest, NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, + codex_turn_id_range, may_carry_title, title_from_response, }, }, }; @@ -32,6 +32,8 @@ const BACKFILL_NAME: &str = "session_titles_v1"; /// Marks the one-time matching of Codex naming threads captured before /// matching existed. const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; +/// Rebuild links after switching to archived request bodies and scoped adapters. +const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; /// Widest delivery delay tolerated between a naming thread and its task when /// looking for candidates, in microseconds. const LINK_DELIVERY_WINDOW_MICROS: i64 = 24 * 60 * 60 * 1_000_000; @@ -90,13 +92,11 @@ fn naming_call( connection: &mut SqliteConnection, event: &CapturedEvent<'_>, ) -> Result, AppError> { - let Some(text) = event.text else { - return Ok(None); - }; let session = event.session_pk.to_string(); match event.event_type { "request" => { - let Some(task) = TitleTask::from_request(text) else { + let Some(naming) = NamingRequest::parse(event.agent_name, event.text, event.metadata) + else { return Ok(None); }; let response = llm_usage_events::table @@ -117,32 +117,33 @@ fn naming_call( .and_then(title_from_response) .map(|title| (title, id, observed_at)) }); - let prompt = codex_naming_prompt(text).map(str::to_owned); Ok(Some(NamingCall { - task, + task: naming.task, title, - prompt, + prompt: naming.prompt, })) } - "response" if may_carry_title(text) => { - let Some(title) = title_from_response(text) else { + "response" if event.text.is_some_and(may_carry_title) => { + let Some(title) = event.text.and_then(title_from_response) else { return Ok(None); }; - let request_text = llm_usage_events::table + let request = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(&session)) .filter(llm_usage_events::turn_index.eq(event.turn_index)) .filter(llm_usage_events::event_type.eq("request")) .filter(llm_usage_events::observed_at.le(event.observed_at)) .order_by(llm_usage_events::observed_at.desc()) - .select(llm_usage_events::text) - .first::>(connection) - .optional()? - .flatten(); - Ok(request_text.as_deref().and_then(|request_text| { - TitleTask::from_request(request_text).map(|task| NamingCall { - task, - title: Some((title, event.event_pk.to_string(), event.observed_at)), - prompt: codex_naming_prompt(request_text).map(str::to_owned), + .select((llm_usage_events::text, llm_usage_events::metadata)) + .first::<(Option, String)>(connection) + .optional()?; + Ok(request.and_then(|(text, metadata)| { + let metadata = serde_json::from_str::(&metadata).ok()?; + NamingRequest::parse(event.agent_name, text.as_deref(), &metadata).map(|naming| { + NamingCall { + task: naming.task, + title: Some((title, event.event_pk.to_string(), event.observed_at)), + prompt: naming.prompt, + } }) })) } @@ -246,28 +247,29 @@ fn match_pending_naming_threads( let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { return Ok(()); }; - let session = event.session_pk.to_string(); let nearby = session_titles::table .filter(session_titles::user_id.eq(event.user_id.to_string())) + .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) .filter(session_titles::title.is_not_null()) + .filter( + session_titles::parent_session_pk + .is_null() + .or(session_titles::parent_session_pk.ne(event.session_pk.to_string())), + ) .filter(session_titles::naming_turn_id.between(low, high)) - .select(( - session_titles::session_pk, - session_titles::naming_prompt, - session_titles::parent_session_pk, - session_titles::title_event_pk, - )) - .load::<(String, Option, Option, Option)>(connection)?; - for (naming_session, prompt, parent, title_event_pk) in nearby { - if !prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { - continue; - } - match parent { - None => match_naming_thread(connection, &naming_session)?, - Some(parent) if parent != session => { - unmatch(connection, &naming_session, &parent, title_event_pk)?; - } - Some(_) => {} + .select((session_titles::session_pk, session_titles::naming_prompt)) + .load::<(String, Option)>(connection)?; + if nearby.is_empty() { + return Ok(()); + } + let Some(request) = CodexRequest::parse(event.metadata) else { + return Ok(()); + }; + for (naming_session, prompt) in nearby { + if prompt.is_some_and(|prompt| request.opens_with_prompt(&prompt)) { + // Recompute the complete candidate set before adding or withdrawing + // a link. A different device or installation is not a conflict. + match_naming_thread(connection, &naming_session)?; } } Ok(()) @@ -306,10 +308,9 @@ fn match_naming_thread( connection: &mut SqliteConnection, naming_session: &str, ) -> Result<(), AppError> { - let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt)) = + let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt, old_parent)) = session_titles::table .find(naming_session) - .filter(session_titles::parent_session_pk.is_null()) .select(( session_titles::user_id, session_titles::title, @@ -317,6 +318,7 @@ fn match_naming_thread( session_titles::title_observed_at, session_titles::naming_turn_id, session_titles::naming_prompt, + session_titles::parent_session_pk, )) .first::<( String, @@ -325,6 +327,7 @@ fn match_naming_thread( Option, Option, Option, + Option, )>(connection) .optional()? else { @@ -338,6 +341,9 @@ fn match_naming_thread( let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { return Ok(()); }; + let Some(naming) = naming_context(connection, naming_session, turn_id.as_deref())? else { + return Ok(()); + }; let device = agent_sessions::table .find(naming_session) .select(agent_sessions::device_context_id) @@ -362,12 +368,28 @@ fn match_naming_thread( .into_iter() .filter(|(_, metadata)| { serde_json::from_str::(metadata) - .is_ok_and(|metadata| opens_with_prompt(&metadata, &prompt)) + .ok() + .and_then(|value| CodexRequest::parse(&value)) + .is_some_and(|request| request.matches(&naming, &prompt)) }) .map(|(session, _)| session) .collect::>(); let mut sessions = sessions.into_iter(); - let (Some(task_session), None) = (sessions.next(), sessions.next()) else { + let task_session = match (sessions.next(), sessions.next()) { + (Some(session), None) => Some(session), + _ => None, + }; + if let Some(parent) = old_parent.as_deref() + && Some(parent) != task_session.as_deref() + { + unmatch( + connection, + naming_session, + parent, + Some(title_event_pk.clone()), + )?; + } + let Some(task_session) = task_session else { return Ok(()); }; diesel::update(session_titles::table.find(naming_session)) @@ -384,11 +406,36 @@ fn match_naming_thread( ) } +/// Reads the naming request's protocol evidence without changing its archive. +fn naming_context( + connection: &mut SqliteConnection, + naming_session: &str, + turn_id: Option<&str>, +) -> Result, AppError> { + let metadata = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(naming_session)) + .filter(llm_usage_events::event_type.eq("request")) + .filter( + sql::>("json_extract(llm_usage_events.metadata, '$.codex_turn_id')") + .eq(turn_id), + ) + .order_by(llm_usage_events::observed_at.desc()) + .select(llm_usage_events::metadata) + .first::(connection) + .optional()?; + let Some(metadata) = metadata else { + return Ok(None); + }; + let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); + Ok(CodexRequest::parse(&metadata)) +} + /// Runs the one-time scans that bring events stored before capture existed up /// to date. pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { run_once(connection, BACKFILL_NAME, backfill_titles)?; - run_once(connection, LINK_BACKFILL_NAME, backfill_links) + run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; + run_once(connection, LINK_REPAIR_NAME, backfill_links) } fn run_once( @@ -421,6 +468,7 @@ fn run_once( fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { let requests = llm_usage_events::table .filter(llm_usage_events::event_type.eq("request")) + .filter(llm_usage_events::agent_name.eq_any(["codex", "claude-code"])) .filter( llm_usage_events::text .like("%You are naming a coding session%") @@ -432,12 +480,24 @@ fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { llm_usage_events::user_id, llm_usage_events::session_pk, llm_usage_events::turn_index, + llm_usage_events::agent_name, llm_usage_events::text, llm_usage_events::metadata, llm_usage_events::observed_at, )) - .load::<(String, String, String, i32, Option, String, i64)>(connection)?; - for (event_pk, user_id, session_pk, turn_index, text, metadata, observed_at) in requests { + .load::<( + String, + String, + String, + i32, + String, + Option, + String, + i64, + )>(connection)?; + for (event_pk, user_id, session_pk, turn_index, agent_name, text, metadata, observed_at) in + requests + { let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); capture( connection, @@ -446,7 +506,7 @@ fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { session_pk: parse_uuid(&session_pk, "event session id")?, event_pk: parse_uuid(&event_pk, "event id")?, turn_index, - agent_name: "", + agent_name: &agent_name, event_type: "request", text: text.as_deref(), metadata: &metadata, @@ -461,14 +521,13 @@ fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { fn backfill_links(connection: &mut SqliteConnection) -> Result<(), AppError> { let naming_sessions = session_titles::table .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .filter(session_titles::naming_turn_id.is_null()) .select(session_titles::session_pk) .load::(connection)?; for naming_session in naming_sessions { let request = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(&naming_session)) .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::text.like("%Generate a concise, single-line task title%")) + .filter(llm_usage_events::agent_name.eq("codex")) .order_by(llm_usage_events::observed_at.desc()) .select((llm_usage_events::text, llm_usage_events::metadata)) .first::<(Option, String)>(connection) @@ -477,10 +536,13 @@ fn backfill_links(connection: &mut SqliteConnection) -> Result<(), AppError> { continue; }; let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); + let Some(naming) = NamingRequest::parse("codex", text.as_deref(), &metadata) else { + continue; + }; diesel::update(session_titles::table.find(&naming_session)) .set(( session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), - session_titles::naming_prompt.eq(text.as_deref().and_then(codex_naming_prompt)), + session_titles::naming_prompt.eq(naming.prompt), )) .execute(connection)?; match_naming_thread(connection, &naming_session)?; @@ -830,8 +892,8 @@ mod tests { .count() .get_result::(&mut database) .expect("count"), - 2, - "titles and Codex links are each scanned once" + 3, + "titles, initial links and repaired links are each scanned once" ); } @@ -883,4 +945,160 @@ mod tests { assert_eq!(parent_of(&mut database, "title-thread"), None); assert_eq!(session_title(&mut database, "second"), (None, None)); } + fn archive(event: &mut Value, source: &str, installation: &str) { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + let turn = event["metadata"]["codex_turn_id"].clone(); + let payload = json!({ + "input": [ + {"role": "developer", "content": "Instructions"}, + {"role": "user", "content": [{"type": "input_text", "text": event["text"]}]} + ], + "client_metadata": {"x-codex-turn-metadata": json!({ + "thread_source": source, "installation_id": installation + }).to_string()} + }); + event["metadata"] = json!({"codex_turn_id": turn, "raw_body": {"parts": [{ + "truncated": false, "content_base64": STANDARD.encode(payload.to_string()) + }]}}); + } + + #[test] + fn raw_codex_links_work_for_every_arrival_order() { + for naming_first in [false, true] { + for response_first in [false, true] { + let mut database = database(); + let mut naming = codex_naming_thread("title-thread"); + archive(&mut naming[0], "thread_title", "installation"); + // Raw-only requests also work when the flattened text is absent. + naming[0]["text"] = Value::Null; + if response_first { + naming.reverse(); + } + let mut task = codex_task("task", TASK_TURN, TASK); + archive(&mut task, "user", "installation"); + if naming_first { + ingest(&mut database, &naming); + ingest(&mut database, &[task]); + } else { + ingest(&mut database, &[task]); + ingest(&mut database, &naming); + } + assert_eq!( + session_title(&mut database, "task").0.as_deref(), + Some("Draw a pelican") + ); + let task_pk = session_pk(&mut database, "task"); + assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); + } + } + } + + #[test] + fn unrelated_devices_installations_and_auxiliary_threads_do_not_withdraw_links() { + let mut database = database(); + let mut naming = codex_naming_thread("title-thread"); + archive(&mut naming[0], "thread_title", "one"); + let mut task = codex_task("task", TASK_TURN, TASK); + archive(&mut task, "user", "one"); + ingest(&mut database, &[task]); + ingest(&mut database, &naming); + let task_pk = session_pk(&mut database, "task"); + for (session, host, source, installation) in [ + ("other-device", "another-host", "user", "one"), + ("other-install", "local", "user", "two"), + ("guardian", "local", "guardian_review", "one"), + ("subagent", "local", "subagent", "one"), + ] { + let mut request = codex_task(session, TASK_TURN, TASK); + request["device"]["host_name"] = json!(host); + archive(&mut request, source, installation); + ingest(&mut database, &[request]); + assert_eq!( + parent_of(&mut database, "title-thread"), + Some(task_pk), + "{session}" + ); + assert_eq!( + session_title(&mut database, "task").0.as_deref(), + Some("Draw a pelican") + ); + } + let mut conflict = codex_task("second-task", TASK_TURN, TASK); + archive(&mut conflict, "user", "one"); + ingest(&mut database, &[conflict]); + assert_eq!(parent_of(&mut database, "title-thread"), None); + assert_eq!(session_title(&mut database, "task").0, None); + } + + #[test] + fn repairs_existing_unlinked_raw_titles_once() { + let mut database = database(); + let mut naming = codex_naming_thread("title-thread"); + archive(&mut naming[0], "thread_title", "one"); + let mut task = codex_task("task", TASK_TURN, TASK); + archive(&mut task, "user", "one"); + ingest(&mut database, &[task]); + ingest(&mut database, &naming); + let task_pk = session_pk(&mut database, "task"); + let naming_pk = session_pk(&mut database, "title-thread"); + diesel::delete(session_titles::table.find(task_pk.to_string())) + .execute(&mut database) + .expect("remove projection"); + diesel::update(session_titles::table.find(naming_pk.to_string())) + .set(session_titles::parent_session_pk.eq(None::)) + .execute(&mut database) + .expect("simulate old raw-body mismatch"); + for name in [super::BACKFILL_NAME, super::LINK_BACKFILL_NAME] { + diesel::insert_into(schema_backfills::table) + .values(( + schema_backfills::name.eq(name), + schema_backfills::completed_at.eq(0_i64), + )) + .execute(&mut database) + .expect("mark older scans completed"); + } + backfill(&mut database).expect("repair"); + assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); + assert_eq!( + session_title(&mut database, "task").0.as_deref(), + Some("Draw a pelican") + ); + backfill(&mut database).expect("idempotent repair"); + assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); + } + + #[test] + fn generic_agents_are_not_classified_by_codex_or_claude_templates() { + let mut database = database(); + for (session, instruction) in [ + ("generic-cc", CLAUDE_NAMING), + ("generic-codex", CODEX_NAMING), + ] { + ingest( + &mut database, + &[ + event( + "cursor", + session, + &format!("{session}-r"), + "2026-09-30T00:00:01Z", + "request", + instruction, + ), + event( + "cursor", + session, + &format!("{session}-s"), + "2026-09-30T00:00:02Z", + "response", + r#"{"title":"Quoted title"}"#, + ), + ], + ); + assert_eq!(session_title(&mut database, session), (None, None)); + } + backfill(&mut database).expect("historical scan follows the same adapter policy"); + assert_eq!(session_title(&mut database, "generic-cc"), (None, None)); + assert_eq!(session_title(&mut database, "generic-codex"), (None, None)); + } } diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 1c70ee4..5059aa9 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -11,6 +11,7 @@ pub mod attachments; pub mod diagnostics; pub mod event_order; pub mod persistence; +mod provider_body; /// Owner-scoped session index contracts. pub mod session_list; pub mod session_title; diff --git a/crates/abyss-backend/src/usage/provider_body.rs b/crates/abyss-backend/src/usage/provider_body.rs new file mode 100644 index 0000000..4fb9732 --- /dev/null +++ b/crates/abyss-backend/src/usage/provider_body.rs @@ -0,0 +1,134 @@ +//! Stateless decoding of archived provider content, shared by projections. + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde_json::Value; + +#[derive(Clone)] +pub struct Message { + pub id: Option, + pub role: String, + pub text: String, +} + +pub fn request_payload(values: &[Value]) -> Option<&Value> { + values + .iter() + .map(|v| v.get("response").unwrap_or(v)) + .find(|v| v.get("input").is_some() || v.get("messages").is_some()) +} + +/// Decode bytes only for projection. The original base64, whitespace, unknown +/// fields, and SSE events remain untouched in event metadata. +pub fn bodies(metadata: &Value) -> Option> { + let parts = metadata.get("raw_body")?.get("parts")?.as_array()?; + let mut values = Vec::new(); + for part in parts { + if part.get("truncated").and_then(Value::as_bool) != Some(false) + || part + .get("content_encoding") + .and_then(Value::as_str) + .is_some_and(|v| v != "identity") + { + return None; + } + let bytes = STANDARD + .decode(part.get("content_base64")?.as_str()?) + .ok()?; + if let Ok(value) = serde_json::from_slice(&bytes) { + values.push(value); + } else { + let text = std::str::from_utf8(&bytes).ok()?; + let mut data = String::new(); + for line in text.lines().chain(std::iter::once("")) { + if line.is_empty() { + if let Ok(value) = serde_json::from_str(&data) { + values.push(value); + } + data.clear(); + } else if let Some(line) = line.strip_prefix("data:") { + if !data.is_empty() { + data.push('\n'); + } + data.push_str(line.strip_prefix(' ').unwrap_or(line)); + } + } + } + } + Some(values) +} + +pub fn request_messages(payload: &Value) -> Vec { + let mut result = Vec::new(); + for (key, role) in [("instructions", "developer"), ("system", "system")] { + if let Some(value) = payload.get(key) { + result.push(Message { + id: None, + role: role.to_owned(), + text: text(value), + }); + } + } + if let Some(input) = payload.get("input").or_else(|| payload.get("messages")) { + if let Some(items) = input.as_array() { + result.extend(items.iter().filter_map(|v| message(v, "request"))); + } else { + result.extend(message(input, "request")); + } + } + result +} + +pub fn message(value: &Value, default_role: &str) -> Option { + let kind = value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + let role = value + .get("role") + .and_then(Value::as_str) + .unwrap_or(default_role); + if role == "tool" || tool(kind) { + return None; + } + let (role, content) = if kind == "reasoning" { + ("reasoning", value.get("summary")?) + } else { + (role, value.get("content").unwrap_or(value)) + }; + Some(Message { + id: value.get("id").and_then(Value::as_str).map(str::to_owned), + role: role.to_owned(), + text: text(content), + }) +} + +pub fn text(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + Value::Array(values) => values + .iter() + .map(text) + .filter(|v| !v.is_empty()) + .collect::>() + .join("\n"), + Value::Object(obj) => { + let kind = obj.get("type").and_then(Value::as_str).unwrap_or_default(); + if tool(kind) { + return String::new(); + } + obj.get("text") + .or_else(|| obj.get("content")) + .or_else(|| obj.get("thinking")) + .or_else(|| obj.get("output_text")) + .map(text) + .unwrap_or_default() + } + _ => String::new(), + } +} + +fn tool(kind: &str) -> bool { + matches!(kind, "tool_use" | "tool_result" | "tool_call") + || kind.ends_with("_call") + || kind.ends_with("_call_output") +} diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index 7474b2d..3127a31 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -16,6 +16,8 @@ use serde::Deserialize; use serde_json::Value; use uuid::Uuid; +use super::provider_body::{bodies, request_messages, request_payload}; + /// Claude Code instruction for naming the session it runs in. const CLAUDE_CODE_NAMING_INSTRUCTION: &str = "You are naming a coding session"; /// Codex instruction for naming a task; Codex sends it from its own thread. @@ -38,16 +40,16 @@ pub enum TitleTask { /// The call names the session it was sent from (Claude Code). NamesOwnSession, /// The call runs in a separate naming session (Codex); that session is - /// auxiliary and the title names the naming session itself. + /// auxiliary and can be linked to the task session it names. AuxiliarySession, } impl TitleTask { /// Recognizes a naming call from its request text. - pub fn from_request(text: &str) -> Option { - if text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) { + pub fn from_request(agent_name: &str, text: &str) -> Option { + if agent_name == "claude-code" && text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) { Some(Self::NamesOwnSession) - } else if text.contains(CODEX_NAMING_INSTRUCTION) { + } else if agent_name == "codex" && text.contains(CODEX_NAMING_INSTRUCTION) { Some(Self::AuxiliarySession) } else { None @@ -55,6 +57,154 @@ impl TitleTask { } } +/// Agent-specific interpretation of a naming request. Other agents use the +/// ordinary session projection, even when they quote a naming instruction. +pub struct NamingRequest { + /// Whether the title belongs to this session or a separate task. + pub task: TitleTask, + /// User prompt quoted by Codex's naming instruction. + pub prompt: Option, +} + +impl NamingRequest { + /// Reads protocol roles from complete bodies, with legacy text compatibility. + pub fn parse(agent_name: &str, text: Option<&str>, metadata: &Value) -> Option { + if !matches!(agent_name, "codex" | "claude-code") { + return None; + } + // Most requests are ordinary work. Avoid decoding their archives just + // to discover that they do not contain a naming instruction. + if text.is_some_and(|text| TitleTask::from_request(agent_name, text).is_none()) { + return None; + } + if agent_name == "claude-code" { + let instruction = if metadata.get("raw_body").is_some() { + let values = bodies(metadata)?; + request_messages(request_payload(&values)?) + .into_iter() + .find(|message| { + matches!(message.role.as_str(), "system" | "developer" | "user") + && message.text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) + })? + .text + } else { + text?.to_owned() + }; + return Some(Self { + task: TitleTask::from_request(agent_name, &instruction)?, + prompt: None, + }); + } + let raw = metadata.get("raw_body").is_some(); + let request = if raw { + Some(CodexRequest::parse(metadata)?) + } else { + None + }; + if request.as_ref().is_some_and(|r| { + r.thread_source + .as_deref() + .is_some_and(|source| source != "thread_title") + }) { + return None; + } + let text = match &request { + Some(request) => request.user_message.as_deref()?, + None => text?, + }; + let task = TitleTask::from_request(agent_name, text)?; + Some(Self { + task, + prompt: (task == TitleTask::AuxiliarySession) + .then(|| codex_naming_prompt(text).map(str::to_owned)) + .flatten(), + }) + } +} + +/// Evidence used only by the Codex adapter. Protocol identity is never inferred +/// from quoted user text, tool output, or model names. +pub struct CodexRequest { + user_message: Option, + installation_id: Option, + thread_source: Option, + started_ms: Option, +} + +impl CodexRequest { + /// Decodes an archived request or the older structured conversation format. + /// An incomplete archive cannot establish or contradict a title link. + pub fn parse(metadata: &Value) -> Option { + let (user_message, context, installation_id) = if metadata.get("raw_body").is_some() { + let values = bodies(metadata)?; + let payload = request_payload(&values)?; + let user_message = request_messages(payload) + .into_iter() + .rev() + .find(|message| message.role == "user") + .map(|message| message.text); + let client = &payload["client_metadata"]; + let context = match client.get("x-codex-turn-metadata") { + Some(encoded) => serde_json::from_str::(encoded.as_str()?).ok()?, + None => client.clone(), + }; + let installation_id = client + .get("x-codex-installation-id") + .and_then(Value::as_str) + .map(str::to_owned); + (user_message, context, installation_id) + } else { + let user_message = metadata + .pointer("/conversation/messages") + .and_then(Value::as_array) + .and_then(|messages| messages.iter().rev().find(|m| m["role"] == "user")) + .and_then(|message| message["text"].as_str()) + .map(str::to_owned); + (user_message, metadata.clone(), None) + }; + Some(Self { + user_message, + installation_id: context + .get("installation_id") + .and_then(Value::as_str) + .map(str::to_owned) + .or(installation_id), + thread_source: context + .get("thread_source") + .and_then(Value::as_str) + .map(str::to_owned), + started_ms: context + .get("turn_started_at_unix_ms") + .and_then(Value::as_u64) + .or_else(|| codex_turn_id(metadata).and_then(turn_started_ms)), + }) + } + + /// Only ordinary user threads can receive a Codex-generated task title. + pub fn opens_with_prompt(&self, prompt: &str) -> bool { + self.thread_source + .as_deref() + .is_none_or(|source| source == "user") + && self + .user_message + .as_deref() + .is_some_and(|text| text.trim_start().starts_with(prompt)) + } + + /// Requires compatible installation identity, turn time and quoted content. + pub fn matches(&self, naming: &Self, prompt: &str) -> bool { + self.opens_with_prompt(prompt) + && match (&self.installation_id, &naming.installation_id) { + (Some(task), Some(title)) => task == title, + _ => true, + } + && match (self.started_ms, naming.started_ms) { + (Some(task), Some(title)) => task.abs_diff(title) <= CODEX_LINK_WINDOW_MS, + _ => false, + } + } +} + #[derive(Deserialize)] struct TitleObject { title: String, @@ -91,7 +241,7 @@ pub fn may_carry_title(text: &str) -> bool { /// Returns the task message a Codex naming request quotes. Codex may cut long /// messages short, so the quote can be a prefix of the task message. pub fn codex_naming_prompt(request_text: &str) -> Option<&str> { - let (_, prompt) = request_text.rsplit_once(CODEX_PROMPT_MARKER)?; + let (_, prompt) = request_text.split_once(CODEX_PROMPT_MARKER)?; Some(prompt.trim()).filter(|prompt| !prompt.is_empty()) } @@ -105,11 +255,7 @@ pub fn codex_turn_id(metadata: &Value) -> Option<&str> { /// time held in the top 48 bits. pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { const TIME_SHIFT: u32 = 80; - let uuid = Uuid::parse_str(turn_id).ok()?; - if uuid.get_version_num() != 7 { - return None; - } - let started_ms = u64::try_from(uuid.as_u128().checked_shr(TIME_SHIFT)?).ok()?; + let started_ms = turn_started_ms(turn_id)?; let low = u128::from(started_ms.saturating_sub(CODEX_LINK_WINDOW_MS)).checked_shl(TIME_SHIFT)?; let high = u128::from(started_ms.saturating_add(CODEX_LINK_WINDOW_MS)) @@ -121,20 +267,11 @@ pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { )) } -/// Returns `true` when a request's latest user message is the task a naming -/// request quoted. -pub fn opens_with_prompt(metadata: &Value, prompt: &str) -> bool { - metadata - .pointer("/conversation/messages") - .and_then(Value::as_array) - .and_then(|messages| { - messages - .iter() - .rev() - .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) - }) - .and_then(|message| message.get("text")?.as_str()) - .is_some_and(|text| text.trim_start().starts_with(prompt)) +fn turn_started_ms(turn_id: &str) -> Option { + let uuid = Uuid::parse_str(turn_id).ok()?; + (uuid.get_version_num() == 7) + .then(|| u64::try_from(uuid.as_u128() >> 80_u32).ok()) + .flatten() } #[cfg(test)] @@ -144,14 +281,23 @@ mod tests { #[test] fn recognizes_official_naming_instructions() { assert_eq!( - TitleTask::from_request("system: …\n\nYou are naming a coding session so the user…"), + TitleTask::from_request( + "claude-code", + "system: …\n\nYou are naming a coding session so the user…" + ), Some(TitleTask::NamesOwnSession) ); assert_eq!( - TitleTask::from_request("Generate a concise, single-line task title of at most 36"), + TitleTask::from_request( + "codex", + "Generate a concise, single-line task title of at most 36" + ), Some(TitleTask::AuxiliarySession) ); - assert_eq!(TitleTask::from_request("please name this session"), None); + assert_eq!( + TitleTask::from_request("codex", "please name this session"), + None + ); } #[test] @@ -216,9 +362,14 @@ mod tests { {"role": "user", "text": "画一只鹈鹕骑自行车，不要联网"}, {"role": "assistant", "text": "好的"}, ]}}); - assert!(opens_with_prompt(&metadata, "画一只鹈鹕")); - assert!(!opens_with_prompt(&metadata, "")); - assert!(!opens_with_prompt(&serde_json::json!({}), "画")); + let request = CodexRequest::parse(&metadata).expect("legacy request"); + assert!(request.opens_with_prompt("画一只鹈鹕")); + assert!(!request.opens_with_prompt("")); + assert!( + !CodexRequest::parse(&serde_json::json!({})) + .expect("legacy") + .opens_with_prompt("画") + ); } #[test] @@ -226,4 +377,130 @@ mod tests { assert!(may_carry_title(" {\"title\": \"x\"}")); assert!(!may_carry_title("The title is x")); } + fn archived_request(payload: &Value, turn_id: &str) -> Value { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + serde_json::json!({ + "codex_turn_id": turn_id, + "raw_body": {"parts": [{"truncated": false, "content_base64": + STANDARD.encode(serde_json::to_vec(payload).expect("json"))}]} + }) + } + + #[test] + fn other_agents_keep_the_default_session_strategy() { + let text = format!("{CLAUDE_CODE_NAMING_INSTRUCTION}\n{CODEX_NAMING_INSTRUCTION}"); + for agent in ["cursor", "openai", "unknown", ""] { + assert!(NamingRequest::parse(agent, Some(&text), &Value::Null).is_none()); + } + assert_eq!( + TitleTask::from_request("codex", CLAUDE_CODE_NAMING_INSTRUCTION), + None + ); + assert_eq!( + TitleTask::from_request("claude-code", CODEX_NAMING_INSTRUCTION), + None + ); + } + + #[test] + fn naming_reads_protocol_roles_and_preserves_nested_prompt_markers() { + let text = format!("{CODEX_NAMING_INSTRUCTION}\nUser prompt:\nExplain User prompt:\nhello"); + let mut payload = serde_json::json!({ + "input": [{"role": "user", "content": [{"type": "input_text", "text": text}]}], + "client_metadata": {"thread_source": "thread_title"} + }); + let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); + assert_eq!( + NamingRequest::parse("codex", None, &metadata) + .expect("naming") + .prompt + .as_deref(), + Some("Explain User prompt:\nhello") + ); + payload["client_metadata"]["thread_source"] = serde_json::json!("user"); + let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); + assert!( + NamingRequest::parse("codex", Some(&text), &metadata).is_none(), + "a user quoting instructions is not a title thread" + ); + payload["client_metadata"]["thread_source"] = serde_json::json!("thread_title"); + payload["input"][0]["role"] = serde_json::json!("assistant"); + let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); + assert!( + NamingRequest::parse("codex", Some(&text), &metadata).is_none(), + "do not fall back to flattened quoted text" + ); + } + + #[test] + fn claude_names_its_own_session_from_raw_instructions() { + let metadata = archived_request( + &serde_json::json!({ + "system": CLAUDE_CODE_NAMING_INSTRUCTION, + "messages": [{"role": "user", "content": "Fix the build"}] + }), + "unused", + ); + assert_eq!( + NamingRequest::parse("claude-code", None, &metadata) + .expect("CC naming") + .task, + TitleTask::NamesOwnSession + ); + } + + #[test] + fn incomplete_raw_bodies_do_not_fall_back_to_legacy_text() { + for part in [ + serde_json::json!({"truncated": true, "content_base64": "e30="}), + serde_json::json!({"truncated": false, "content_base64": "invalid"}), + serde_json::json!({"truncated": false, "content_encoding": "gzip", "content_base64": "e30="}), + ] { + let metadata = serde_json::json!({"raw_body": {"parts": [part]}, + "conversation": {"messages": [{"role": "user", "text": "hello"}]}}); + assert!(CodexRequest::parse(&metadata).is_none()); + assert!( + NamingRequest::parse("codex", Some(CODEX_NAMING_INSTRUCTION), &metadata).is_none() + ); + } + } + + #[test] + fn codex_matches_installation_source_and_explicit_turn_time() { + let naming = CodexRequest::parse(&serde_json::json!({ + "installation_id": "one", "thread_source": "thread_title", "turn_started_at_unix_ms": 100_000_u64 + })).expect("title"); + let mut metadata = serde_json::json!({ + "codex_turn_id": "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb", + "installation_id": "one", "thread_source": "user", "turn_started_at_unix_ms": 99_950_u64, + "conversation": {"messages": [{"role": "user", "text": "hello there"}]} + }); + assert!( + CodexRequest::parse(&metadata) + .expect("task") + .matches(&naming, "hello") + ); + metadata["installation_id"] = serde_json::json!("two"); + assert!( + !CodexRequest::parse(&metadata) + .expect("task") + .matches(&naming, "hello") + ); + metadata["installation_id"] = serde_json::json!("one"); + for source in ["guardian_review", "subagent", "thread_title"] { + metadata["thread_source"] = serde_json::json!(source); + assert!( + !CodexRequest::parse(&metadata) + .expect("task") + .matches(&naming, "hello") + ); + } + metadata["thread_source"] = serde_json::json!("user"); + metadata["turn_started_at_unix_ms"] = serde_json::json!(160_001_u64); + assert!( + !CodexRequest::parse(&metadata) + .expect("task") + .matches(&naming, "hello") + ); + } } diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs index 27cb03c..57c779d 100644 --- a/crates/abyss-backend/src/usage/trajectory/raw_body.rs +++ b/crates/abyss-backend/src/usage/trajectory/raw_body.rs @@ -2,17 +2,12 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use base64::{Engine as _, engine::general_purpose::STANDARD}; use serde_json::Value; use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; - -#[derive(Clone)] -struct Message { - id: Option, - role: String, - text: String, -} +use crate::usage::provider_body::{ + Message, bodies, message, request_messages, request_payload, text, +}; impl Message { fn matches(&self, other: &Self) -> bool { @@ -52,7 +47,7 @@ impl RawProjection { stream: &mut usize, stream_ids: &mut HashMap, ) -> Option> { - let values = bodies(event)?; + let values = bodies(&event.metadata)?; let request = event.event_type == "request"; let (messages, incremental) = if request { let payload = request_payload(&values)?; @@ -137,129 +132,6 @@ impl RawProjection { } } -fn request_payload(values: &[Value]) -> Option<&Value> { - values - .iter() - .map(|v| v.get("response").unwrap_or(v)) - .find(|v| v.get("input").is_some() || v.get("messages").is_some()) -} - -/// Decode bytes only for projection. The original base64, whitespace, unknown -/// fields, and SSE events remain untouched in event metadata. -fn bodies(event: &UsageEventResponse) -> Option> { - let parts = event.metadata.get("raw_body")?.get("parts")?.as_array()?; - let mut values = Vec::new(); - for part in parts { - if part.get("truncated").and_then(Value::as_bool) != Some(false) - || part - .get("content_encoding") - .and_then(Value::as_str) - .is_some_and(|v| v != "identity") - { - return None; - } - let bytes = STANDARD - .decode(part.get("content_base64")?.as_str()?) - .ok()?; - if let Ok(value) = serde_json::from_slice(&bytes) { - values.push(value); - } else { - let text = std::str::from_utf8(&bytes).ok()?; - let mut data = String::new(); - for line in text.lines().chain(std::iter::once("")) { - if line.is_empty() { - if let Ok(value) = serde_json::from_str(&data) { - values.push(value); - } - data.clear(); - } else if let Some(line) = line.strip_prefix("data:") { - if !data.is_empty() { - data.push('\n'); - } - data.push_str(line.strip_prefix(' ').unwrap_or(line)); - } - } - } - } - Some(values) -} - -fn request_messages(payload: &Value) -> Vec { - let mut result = Vec::new(); - for (key, role) in [("instructions", "developer"), ("system", "system")] { - if let Some(value) = payload.get(key) { - result.push(Message { - id: None, - role: role.to_owned(), - text: text(value), - }); - } - } - if let Some(input) = payload.get("input").or_else(|| payload.get("messages")) { - if let Some(items) = input.as_array() { - result.extend(items.iter().filter_map(|v| message(v, "request"))); - } else { - result.extend(message(input, "request")); - } - } - result -} - -fn message(value: &Value, default_role: &str) -> Option { - let kind = value - .get("type") - .and_then(Value::as_str) - .unwrap_or_default(); - let role = value - .get("role") - .and_then(Value::as_str) - .unwrap_or(default_role); - if role == "tool" || tool(kind) { - return None; - } - let (role, content) = if kind == "reasoning" { - ("reasoning", value.get("summary")?) - } else { - (role, value.get("content").unwrap_or(value)) - }; - Some(Message { - id: value.get("id").and_then(Value::as_str).map(str::to_owned), - role: role.to_owned(), - text: text(content), - }) -} - -fn text(value: &Value) -> String { - match value { - Value::String(value) => value.clone(), - Value::Array(values) => values - .iter() - .map(text) - .filter(|v| !v.is_empty()) - .collect::>() - .join("\n"), - Value::Object(obj) => { - let kind = obj.get("type").and_then(Value::as_str).unwrap_or_default(); - if tool(kind) { - return String::new(); - } - obj.get("text") - .or_else(|| obj.get("content")) - .or_else(|| obj.get("thinking")) - .or_else(|| obj.get("output_text")) - .map(text) - .unwrap_or_default() - } - _ => String::new(), - } -} - -fn tool(kind: &str) -> bool { - matches!(kind, "tool_use" | "tool_result" | "tool_call") - || kind.ends_with("_call") - || kind.ends_with("_call_output") -} - fn response_messages(values: &[Value]) -> Vec { let mut items = BTreeMap::new(); let mut anthropic_blocks = BTreeMap::::new(); diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index a55bf85..7e932e1 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -374,6 +374,9 @@ def run(base_url: str, token: str, run_id: str) -> None: "search must return structured highlights", ) + from session_titles_api import run_title_contract + + run_title_contract(base_url, token, run_id) print(f"blackbox API contract passed for {session_id}") diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py new file mode 100644 index 0000000..b3af098 --- /dev/null +++ b/scripts/tests/session_titles_api.py @@ -0,0 +1,83 @@ +"""Session title adapter contract shared by both storage black-box suites.""" + +import base64 +import json +import urllib.parse +import uuid + +from blackbox_api import event, request, require, require_status + + +NAMING_TURN = "01a0f320-138e-7d21-99ea-03a7ae2a9b50" +TASK_TURN = "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb" +PROMPT = "Draw a pelican riding a bicycle" +CODEX_NAMING = "Generate a concise, single-line task title\nUser prompt:\n" + PROMPT +CLAUDE_NAMING = "You are naming a coding session" + + +def run_title_contract(base_url: str, token: str, run_id: str) -> None: + def make(session, kind, text, *, source="user", installation="one", agent="codex"): + value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, {}, run_id) + value.pop("attachments", None) + value["agent"]["name"] = agent + value["device"]["host_name"] = f"title-contract-{run_id}" + value["observed_at"] = "2026-09-30T00:00:01Z" if kind == "request" else "2026-09-30T00:00:02Z" + value["metadata"] = {"codex_turn_id": NAMING_TURN if source == "thread_title" else TASK_TURN} + if kind == "request": + payload = { + "input": [{"role": "user", "content": [{"type": "input_text", "text": text}]}], + "client_metadata": {"x-codex-turn-metadata": json.dumps({ + "thread_source": source, "installation_id": installation, + })}, + } + value["metadata"]["raw_body"] = {"parts": [{ + "truncated": False, + "content_base64": base64.b64encode(json.dumps(payload).encode()).decode(), + }]} + return value + + def ingest(*events): + result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) + require_status(result, 200, "title fixture ingest") + + def session(name): + query = urllib.parse.urlencode({"session_id": f"{run_id}-{name}", "limit": 1}) + result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) + require_status(result, 200, "title event lookup") + pk = result.json()["events"][0]["session_pk"] + result = request(base_url, "GET", f"/v1/agent-usage/sessions/{pk}/outline", token=token) + require_status(result, 200, "title outline") + return result.json()["session"] + + # Response-first and raw-only naming requests exercise both storage paths. + naming = make("naming", "request", CODEX_NAMING, source="thread_title") + naming["text"] = None + ingest(make("naming", "response", '{"title":"Draw a pelican"}', source="thread_title")) + ingest(naming) + ingest(make("task", "request", PROMPT)) + task = session("task") + require(task["title"] == "Draw a pelican", "raw Codex task receives its generated title") + require(session("naming")["parent_session_pk"] == task["session_pk"], "title source links to the task") + + other_device = make("other-device", "request", PROMPT) + other_device["device"]["host_name"] += "-other" + ingest(other_device, make("other-installation", "request", PROMPT, installation="two"), + make("guardian", "request", PROMPT, source="guardian_review")) + require(session("naming")["parent_session_pk"] == task["session_pk"], "unrelated requests do not withdraw the link") + + ingest(make("conflict", "request", PROMPT)) + require(session("naming")["parent_session_pk"] is None, "a real second candidate withdraws the link") + require(session("task")["title"] is None, "ambiguous title is removed from its former task") + + for agent, name, instruction, expected in [ + ("claude-code", "claude", CLAUDE_NAMING, "Build fix"), + ("cursor", "generic-claude", CLAUDE_NAMING, None), + ("cursor", "generic-codex", CODEX_NAMING, None), + ("codex", "quoted-user", CODEX_NAMING, None), + ]: + ingest(make(name, "request", instruction, agent=agent), + make(name, "response", '{"title":"Build fix"}', agent=agent)) + value = session(name) + require(value["title"] == expected, f"{agent}/{name} follows its adapter policy") + require(value["purpose"] is None, f"{agent}/{name} is not an auxiliary title thread") + print("session title adapter API contract passed") From 7f4f996d7d63bd4e587a580d85f13f037df0134a Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 00:48:45 -0700 Subject: [PATCH 14/30] refactor: simplify Codex title matching --- README.md | 2 +- .../src/storage/postgres/session_titles.rs | 38 +--- .../src/storage/sqlite/session_titles.rs | 73 ++----- .../abyss-backend/src/usage/session_title.rs | 201 ++++-------------- scripts/tests/session_titles_api.py | 7 +- 5 files changed, 76 insertions(+), 245 deletions(-) diff --git a/README.md b/README.md index f44a85a..986d8b5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ and optional full-text search. Session titles have dedicated adapters for Codex and Claude Code. Claude Code names its own session; Codex names a separate task thread, matched by quoted -user content, turn start time, device, and available installation identity. +user content, turn start time, and device. Only a unique task candidate receives the generated title. Other Agents keep the default session identity and are not classified using these Agents' naming templates. Both adapters read archived protocol messages and support older diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index 78cea15..bbde8d1 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -18,8 +18,8 @@ use crate::{ usage::{ SessionInfo, session_title::{ - CodexRequest, NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, - codex_turn_id_range, may_carry_title, title_from_response, + NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, + may_carry_title, opens_with_prompt, title_from_response, }, }, }; @@ -258,13 +258,10 @@ fn match_pending_naming_threads( if nearby.is_empty() { return Ok(()); } - let Some(request) = CodexRequest::parse(event.metadata) else { - return Ok(()); - }; for (naming_session, prompt) in nearby { - if prompt.is_some_and(|prompt| request.opens_with_prompt(&prompt)) { + if prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { // Recompute the complete candidate set before adding or withdrawing - // a link. A different device or installation is not a conflict. + // a link. A request from a different device is not a conflict. match_naming_thread(connection, naming_session)?; } } @@ -344,9 +341,6 @@ fn match_naming_thread( ) else { return Ok(()); }; - let Some(naming) = naming_context(connection, naming_session, turn_id.as_deref())? else { - return Ok(()); - }; let device = agent_sessions::table .find(naming_session) .select(agent_sessions::device_context_id) @@ -366,9 +360,7 @@ fn match_naming_thread( .load::<(Uuid, Value)>(connection)?; let sessions = candidates .into_iter() - .filter(|(_, metadata)| { - CodexRequest::parse(metadata).is_some_and(|request| request.matches(&naming, &prompt)) - }) + .filter(|(_, metadata)| opens_with_prompt(metadata, &prompt)) .map(|(session, _)| session) .collect::>(); let mut sessions = sessions.into_iter(); @@ -398,26 +390,6 @@ fn match_naming_thread( ) } -/// Reads the naming request's protocol evidence without changing its archive. -fn naming_context( - connection: &mut PgConnection, - naming_session: Uuid, - turn_id: Option<&str>, -) -> Result, AppError> { - let metadata = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(naming_session)) - .filter(llm_usage_events::event_type.eq("request")) - .filter(sql::>("llm_usage_events.metadata ->> 'codex_turn_id'").eq(turn_id)) - .order_by(llm_usage_events::observed_at.desc()) - .select(llm_usage_events::metadata) - .first::(connection) - .optional()?; - let Some(metadata) = metadata else { - return Ok(None); - }; - Ok(CodexRequest::parse(&metadata)) -} - /// Runs the one-time scans that bring events stored before capture existed up /// to date. pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index b303f9c..5c1f589 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -21,8 +21,8 @@ use crate::{ usage::{ SessionInfo, session_title::{ - CodexRequest, NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, - codex_turn_id_range, may_carry_title, title_from_response, + NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, + may_carry_title, opens_with_prompt, title_from_response, }, }, }; @@ -262,13 +262,10 @@ fn match_pending_naming_threads( if nearby.is_empty() { return Ok(()); } - let Some(request) = CodexRequest::parse(event.metadata) else { - return Ok(()); - }; for (naming_session, prompt) in nearby { - if prompt.is_some_and(|prompt| request.opens_with_prompt(&prompt)) { + if prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { // Recompute the complete candidate set before adding or withdrawing - // a link. A different device or installation is not a conflict. + // a link. A request from a different device is not a conflict. match_naming_thread(connection, &naming_session)?; } } @@ -341,9 +338,6 @@ fn match_naming_thread( let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { return Ok(()); }; - let Some(naming) = naming_context(connection, naming_session, turn_id.as_deref())? else { - return Ok(()); - }; let device = agent_sessions::table .find(naming_session) .select(agent_sessions::device_context_id) @@ -368,9 +362,7 @@ fn match_naming_thread( .into_iter() .filter(|(_, metadata)| { serde_json::from_str::(metadata) - .ok() - .and_then(|value| CodexRequest::parse(&value)) - .is_some_and(|request| request.matches(&naming, &prompt)) + .is_ok_and(|metadata| opens_with_prompt(&metadata, &prompt)) }) .map(|(session, _)| session) .collect::>(); @@ -406,30 +398,6 @@ fn match_naming_thread( ) } -/// Reads the naming request's protocol evidence without changing its archive. -fn naming_context( - connection: &mut SqliteConnection, - naming_session: &str, - turn_id: Option<&str>, -) -> Result, AppError> { - let metadata = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(naming_session)) - .filter(llm_usage_events::event_type.eq("request")) - .filter( - sql::>("json_extract(llm_usage_events.metadata, '$.codex_turn_id')") - .eq(turn_id), - ) - .order_by(llm_usage_events::observed_at.desc()) - .select(llm_usage_events::metadata) - .first::(connection) - .optional()?; - let Some(metadata) = metadata else { - return Ok(None); - }; - let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); - Ok(CodexRequest::parse(&metadata)) -} - /// Runs the one-time scans that bring events stored before capture existed up /// to date. pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { @@ -945,7 +913,7 @@ mod tests { assert_eq!(parent_of(&mut database, "title-thread"), None); assert_eq!(session_title(&mut database, "second"), (None, None)); } - fn archive(event: &mut Value, source: &str, installation: &str) { + fn archive(event: &mut Value, source: &str) { use base64::{Engine as _, engine::general_purpose::STANDARD}; let turn = event["metadata"]["codex_turn_id"].clone(); let payload = json!({ @@ -954,7 +922,7 @@ mod tests { {"role": "user", "content": [{"type": "input_text", "text": event["text"]}]} ], "client_metadata": {"x-codex-turn-metadata": json!({ - "thread_source": source, "installation_id": installation + "thread_source": source }).to_string()} }); event["metadata"] = json!({"codex_turn_id": turn, "raw_body": {"parts": [{ @@ -968,14 +936,14 @@ mod tests { for response_first in [false, true] { let mut database = database(); let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title", "installation"); + archive(&mut naming[0], "thread_title"); // Raw-only requests also work when the flattened text is absent. naming[0]["text"] = Value::Null; if response_first { naming.reverse(); } let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user", "installation"); + archive(&mut task, "user"); if naming_first { ingest(&mut database, &naming); ingest(&mut database, &[task]); @@ -994,24 +962,23 @@ mod tests { } #[test] - fn unrelated_devices_installations_and_auxiliary_threads_do_not_withdraw_links() { + fn other_devices_and_auxiliary_threads_do_not_withdraw_links() { let mut database = database(); let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title", "one"); + archive(&mut naming[0], "thread_title"); let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user", "one"); + archive(&mut task, "user"); ingest(&mut database, &[task]); ingest(&mut database, &naming); let task_pk = session_pk(&mut database, "task"); - for (session, host, source, installation) in [ - ("other-device", "another-host", "user", "one"), - ("other-install", "local", "user", "two"), - ("guardian", "local", "guardian_review", "one"), - ("subagent", "local", "subagent", "one"), + for (session, host, source) in [ + ("other-device", "another-host", "user"), + ("guardian", "local", "guardian_review"), + ("subagent", "local", "subagent"), ] { let mut request = codex_task(session, TASK_TURN, TASK); request["device"]["host_name"] = json!(host); - archive(&mut request, source, installation); + archive(&mut request, source); ingest(&mut database, &[request]); assert_eq!( parent_of(&mut database, "title-thread"), @@ -1024,7 +991,7 @@ mod tests { ); } let mut conflict = codex_task("second-task", TASK_TURN, TASK); - archive(&mut conflict, "user", "one"); + archive(&mut conflict, "user"); ingest(&mut database, &[conflict]); assert_eq!(parent_of(&mut database, "title-thread"), None); assert_eq!(session_title(&mut database, "task").0, None); @@ -1034,9 +1001,9 @@ mod tests { fn repairs_existing_unlinked_raw_titles_once() { let mut database = database(); let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title", "one"); + archive(&mut naming[0], "thread_title"); let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user", "one"); + archive(&mut task, "user"); ingest(&mut database, &[task]); ingest(&mut database, &naming); let task_pk = session_pk(&mut database, "task"); diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index 3127a31..f35cd72 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -95,114 +95,54 @@ impl NamingRequest { prompt: None, }); } - let raw = metadata.get("raw_body").is_some(); - let request = if raw { - Some(CodexRequest::parse(metadata)?) + let text = if metadata.get("raw_body").is_some() { + codex_user_message(metadata, "thread_title")? } else { - None + text?.to_owned() }; - if request.as_ref().is_some_and(|r| { - r.thread_source - .as_deref() - .is_some_and(|source| source != "thread_title") - }) { - return None; - } - let text = match &request { - Some(request) => request.user_message.as_deref()?, - None => text?, - }; - let task = TitleTask::from_request(agent_name, text)?; Some(Self { - task, - prompt: (task == TitleTask::AuxiliarySession) - .then(|| codex_naming_prompt(text).map(str::to_owned)) - .flatten(), + task: TitleTask::from_request(agent_name, &text)?, + prompt: codex_naming_prompt(&text).map(str::to_owned), }) } } -/// Evidence used only by the Codex adapter. Protocol identity is never inferred -/// from quoted user text, tool output, or model names. -pub struct CodexRequest { - user_message: Option, - installation_id: Option, - thread_source: Option, - started_ms: Option, -} - -impl CodexRequest { - /// Decodes an archived request or the older structured conversation format. - /// An incomplete archive cannot establish or contradict a title link. - pub fn parse(metadata: &Value) -> Option { - let (user_message, context, installation_id) = if metadata.get("raw_body").is_some() { - let values = bodies(metadata)?; - let payload = request_payload(&values)?; - let user_message = request_messages(payload) - .into_iter() - .rev() - .find(|message| message.role == "user") - .map(|message| message.text); - let client = &payload["client_metadata"]; - let context = match client.get("x-codex-turn-metadata") { - Some(encoded) => serde_json::from_str::(encoded.as_str()?).ok()?, - None => client.clone(), - }; - let installation_id = client - .get("x-codex-installation-id") - .and_then(Value::as_str) - .map(str::to_owned); - (user_message, context, installation_id) - } else { - let user_message = metadata - .pointer("/conversation/messages") - .and_then(Value::as_array) - .and_then(|messages| messages.iter().rev().find(|m| m["role"] == "user")) - .and_then(|message| message["text"].as_str()) - .map(str::to_owned); - (user_message, metadata.clone(), None) +/// Reads the last user message from a Codex request. The protocol source, when +/// present, distinguishes title tasks from ordinary work and auxiliary calls. +fn codex_user_message(metadata: &Value, source: &str) -> Option { + if metadata.get("raw_body").is_some() { + let values = bodies(metadata)?; + let payload = request_payload(&values)?; + let client = &payload["client_metadata"]; + let context = match client.get("x-codex-turn-metadata") { + Some(encoded) => serde_json::from_str::(encoded.as_str()?).ok()?, + None => client.clone(), }; - Some(Self { - user_message, - installation_id: context - .get("installation_id") - .and_then(Value::as_str) - .map(str::to_owned) - .or(installation_id), - thread_source: context - .get("thread_source") - .and_then(Value::as_str) - .map(str::to_owned), - started_ms: context - .get("turn_started_at_unix_ms") - .and_then(Value::as_u64) - .or_else(|| codex_turn_id(metadata).and_then(turn_started_ms)), - }) - } - - /// Only ordinary user threads can receive a Codex-generated task title. - pub fn opens_with_prompt(&self, prompt: &str) -> bool { - self.thread_source - .as_deref() - .is_none_or(|source| source == "user") - && self - .user_message - .as_deref() - .is_some_and(|text| text.trim_start().starts_with(prompt)) + if context["thread_source"] + .as_str() + .is_some_and(|value| value != source) + { + return None; + } + return request_messages(payload) + .into_iter() + .rev() + .find(|message| message.role == "user") + .map(|message| message.text); } + metadata + .pointer("/conversation/messages")? + .as_array()? + .iter() + .rev() + .find(|message| message["role"] == "user")?["text"] + .as_str() + .map(str::to_owned) +} - /// Requires compatible installation identity, turn time and quoted content. - pub fn matches(&self, naming: &Self, prompt: &str) -> bool { - self.opens_with_prompt(prompt) - && match (&self.installation_id, &naming.installation_id) { - (Some(task), Some(title)) => task == title, - _ => true, - } - && match (self.started_ms, naming.started_ms) { - (Some(task), Some(title)) => task.abs_diff(title) <= CODEX_LINK_WINDOW_MS, - _ => false, - } - } +/// A naming prompt may be a truncated prefix of the task's user message. +pub fn opens_with_prompt(metadata: &Value, prompt: &str) -> bool { + codex_user_message(metadata, "user").is_some_and(|text| text.trim_start().starts_with(prompt)) } #[derive(Deserialize)] @@ -255,7 +195,11 @@ pub fn codex_turn_id(metadata: &Value) -> Option<&str> { /// time held in the top 48 bits. pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { const TIME_SHIFT: u32 = 80; - let started_ms = turn_started_ms(turn_id)?; + let uuid = Uuid::parse_str(turn_id).ok()?; + if uuid.get_version_num() != 7 { + return None; + } + let started_ms = u64::try_from(uuid.as_u128().checked_shr(TIME_SHIFT)?).ok()?; let low = u128::from(started_ms.saturating_sub(CODEX_LINK_WINDOW_MS)).checked_shl(TIME_SHIFT)?; let high = u128::from(started_ms.saturating_add(CODEX_LINK_WINDOW_MS)) @@ -267,13 +211,6 @@ pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { )) } -fn turn_started_ms(turn_id: &str) -> Option { - let uuid = Uuid::parse_str(turn_id).ok()?; - (uuid.get_version_num() == 7) - .then(|| u64::try_from(uuid.as_u128() >> 80_u32).ok()) - .flatten() -} - #[cfg(test)] mod tests { use super::*; @@ -362,14 +299,9 @@ mod tests { {"role": "user", "text": "画一只鹈鹕骑自行车，不要联网"}, {"role": "assistant", "text": "好的"}, ]}}); - let request = CodexRequest::parse(&metadata).expect("legacy request"); - assert!(request.opens_with_prompt("画一只鹈鹕")); - assert!(!request.opens_with_prompt("")); - assert!( - !CodexRequest::parse(&serde_json::json!({})) - .expect("legacy") - .opens_with_prompt("画") - ); + assert!(opens_with_prompt(&metadata, "画一只鹈鹕")); + assert!(!opens_with_prompt(&metadata, "")); + assert!(!opens_with_prompt(&serde_json::json!({}), "画")); } #[test] @@ -458,49 +390,10 @@ mod tests { ] { let metadata = serde_json::json!({"raw_body": {"parts": [part]}, "conversation": {"messages": [{"role": "user", "text": "hello"}]}}); - assert!(CodexRequest::parse(&metadata).is_none()); + assert!(!opens_with_prompt(&metadata, "hello")); assert!( NamingRequest::parse("codex", Some(CODEX_NAMING_INSTRUCTION), &metadata).is_none() ); } } - - #[test] - fn codex_matches_installation_source_and_explicit_turn_time() { - let naming = CodexRequest::parse(&serde_json::json!({ - "installation_id": "one", "thread_source": "thread_title", "turn_started_at_unix_ms": 100_000_u64 - })).expect("title"); - let mut metadata = serde_json::json!({ - "codex_turn_id": "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb", - "installation_id": "one", "thread_source": "user", "turn_started_at_unix_ms": 99_950_u64, - "conversation": {"messages": [{"role": "user", "text": "hello there"}]} - }); - assert!( - CodexRequest::parse(&metadata) - .expect("task") - .matches(&naming, "hello") - ); - metadata["installation_id"] = serde_json::json!("two"); - assert!( - !CodexRequest::parse(&metadata) - .expect("task") - .matches(&naming, "hello") - ); - metadata["installation_id"] = serde_json::json!("one"); - for source in ["guardian_review", "subagent", "thread_title"] { - metadata["thread_source"] = serde_json::json!(source); - assert!( - !CodexRequest::parse(&metadata) - .expect("task") - .matches(&naming, "hello") - ); - } - metadata["thread_source"] = serde_json::json!("user"); - metadata["turn_started_at_unix_ms"] = serde_json::json!(160_001_u64); - assert!( - !CodexRequest::parse(&metadata) - .expect("task") - .matches(&naming, "hello") - ); - } } diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py index b3af098..683c7c1 100644 --- a/scripts/tests/session_titles_api.py +++ b/scripts/tests/session_titles_api.py @@ -16,7 +16,7 @@ def run_title_contract(base_url: str, token: str, run_id: str) -> None: - def make(session, kind, text, *, source="user", installation="one", agent="codex"): + def make(session, kind, text, *, source="user", agent="codex"): value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, {}, run_id) value.pop("attachments", None) value["agent"]["name"] = agent @@ -27,7 +27,7 @@ def make(session, kind, text, *, source="user", installation="one", agent="codex payload = { "input": [{"role": "user", "content": [{"type": "input_text", "text": text}]}], "client_metadata": {"x-codex-turn-metadata": json.dumps({ - "thread_source": source, "installation_id": installation, + "thread_source": source, })}, } value["metadata"]["raw_body"] = {"parts": [{ @@ -61,8 +61,7 @@ def session(name): other_device = make("other-device", "request", PROMPT) other_device["device"]["host_name"] += "-other" - ingest(other_device, make("other-installation", "request", PROMPT, installation="two"), - make("guardian", "request", PROMPT, source="guardian_review")) + ingest(other_device, make("guardian", "request", PROMPT, source="guardian_review")) require(session("naming")["parent_session_pk"] == task["session_pk"], "unrelated requests do not withdraw the link") ingest(make("conflict", "request", PROMPT)) From 20dd3b26973db8c979dd113bf853cc0e6f8cb70c Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 01:11:06 -0700 Subject: [PATCH 15/30] fix: use first user input for default session titles --- README.md | 7 +- .../src/storage/postgres/session_titles.rs | 114 ++++++++++- .../src/storage/sqlite/session_titles.rs | 188 +++++++++++++++++- crates/abyss-backend/src/usage/mod.rs | 2 +- .../abyss-backend/src/usage/session_list.rs | 2 +- .../abyss-backend/src/usage/session_title.rs | 84 +++++++- scripts/tests/session_titles_api.py | 23 ++- 7 files changed, 406 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 986d8b5..7bf6865 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,11 @@ and optional full-text search. Session titles have dedicated adapters for Codex and Claude Code. Claude Code names its own session; Codex names a separate task thread, matched by quoted user content, turn start time, and device. -Only a unique task candidate receives the generated title. Other Agents keep -the default session identity and are not classified using these Agents' naming -templates. Both adapters read archived protocol messages and support older +Only a unique task candidate receives the generated title. Other Agents use +their first nonempty user input, collapsed to one line and truncated to 80 +characters with an ellipsis. Both adapters read archived protocol messages and support older structured event records. Auxiliary title calls retain their own events and usage. +Existing sessions receive default titles through a one-time backfill. Exactly one storage profile is compiled into a binary: diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index bbde8d1..2311539 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -19,7 +19,7 @@ use crate::{ SessionInfo, session_title::{ NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, - may_carry_title, opens_with_prompt, title_from_response, + default_title, may_carry_title, opens_with_prompt, title_from_response, }, }, }; @@ -31,6 +31,8 @@ const BACKFILL_NAME: &str = "session_titles_v1"; const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; /// Rebuild links after switching to archived request bodies and scoped adapters. const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; +/// Names existing sessions from other Agents using their first user input. +const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; /// Widest delivery delay tolerated between a naming thread and its task when /// looking for candidates, in hours. const LINK_DELIVERY_WINDOW_HOURS: i64 = 24; @@ -79,12 +81,57 @@ pub(super) fn capture( connection: &mut PgConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { + if !matches!(event.agent_name, "codex" | "claude-code") { + return capture_default(connection, event); + } match naming_call(connection, event)? { Some(call) => store(connection, event, call), None => match_pending_naming_threads(connection, event), } } +/// Default sessions keep the earliest captured user input, even after late delivery. +fn capture_default( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + if event.event_type != "request" { + return Ok(()); + } + let session = event.session_pk; + let stored_at = session_titles::table + .find(session) + .select(session_titles::title_observed_at) + .first::>>(connection) + .optional()? + .flatten(); + if stored_at.is_some_and(|at| at <= event.observed_at) { + return Ok(()); + } + let Some(title) = default_title(event.metadata) else { + return Ok(()); + }; + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(event.user_id), + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(Utc::now()), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(Utc::now()), + )) + .execute(connection)?; + Ok(()) +} + fn naming_call( connection: &mut PgConnection, event: &CapturedEvent<'_>, @@ -395,7 +442,8 @@ fn match_naming_thread( pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { run_once(connection, BACKFILL_NAME, backfill_titles)?; run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; - run_once(connection, LINK_REPAIR_NAME, backfill_links) + run_once(connection, LINK_REPAIR_NAME, backfill_links)?; + run_once(connection, DEFAULT_BACKFILL_NAME, backfill_defaults) } fn run_once( @@ -508,6 +556,68 @@ fn backfill_links(connection: &mut PgConnection) -> Result<(), AppError> { Ok(()) } +/// Visit requests in time order until the first readable user input is found. +fn backfill_defaults(connection: &mut PgConnection) -> Result<(), AppError> { + let sessions = agent_sessions::table + .filter(agent_sessions::agent_name.ne_all(["codex", "claude-code"])) + .select(( + agent_sessions::id, + agent_sessions::user_id, + agent_sessions::agent_name, + )) + .load::<(Uuid, Uuid, String)>(connection)?; + for (session, user_id, agent_name) in sessions { + let mut offset = 0_i64; + loop { + let requests = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(session)) + .filter(llm_usage_events::event_type.eq("request")) + .order_by(( + llm_usage_events::observed_at.asc(), + llm_usage_events::id.asc(), + )) + .offset(offset) + .limit(16) + .select(( + llm_usage_events::id, + llm_usage_events::observed_at, + llm_usage_events::metadata, + )) + .load::<(Uuid, DateTime, Value)>(connection)?; + if requests.is_empty() { + break; + } + let mut found = false; + for (event_pk, observed_at, metadata) in requests { + if default_title(&metadata).is_none() { + continue; + } + capture_default( + connection, + &CapturedEvent { + user_id, + session_pk: session, + event_pk, + turn_index: 0, + agent_name: &agent_name, + event_type: "request", + text: None, + metadata: &metadata, + observed_at, + }, + )?; + found = true; + break; + } + if found { + break; + } + offset = offset.saturating_add(16); + } + } + Ok(()) +} + /// Adds the stored title, purpose, and matched task session to a session. pub(super) fn fill( connection: &mut PgConnection, diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index 5c1f589..59582fa 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -22,7 +22,7 @@ use crate::{ SessionInfo, session_title::{ NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, - may_carry_title, opens_with_prompt, title_from_response, + default_title, may_carry_title, opens_with_prompt, title_from_response, }, }, }; @@ -34,6 +34,8 @@ const BACKFILL_NAME: &str = "session_titles_v1"; const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; /// Rebuild links after switching to archived request bodies and scoped adapters. const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; +/// Names existing sessions from other Agents using their first user input. +const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; /// Widest delivery delay tolerated between a naming thread and its task when /// looking for candidates, in microseconds. const LINK_DELIVERY_WINDOW_MICROS: i64 = 24 * 60 * 60 * 1_000_000; @@ -82,12 +84,57 @@ pub(super) fn capture( connection: &mut SqliteConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { + if !matches!(event.agent_name, "codex" | "claude-code") { + return capture_default(connection, event); + } match naming_call(connection, event)? { Some(call) => store(connection, event, call), None => match_pending_naming_threads(connection, event), } } +/// Default sessions keep the earliest captured user input, even after late delivery. +fn capture_default( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + if event.event_type != "request" { + return Ok(()); + } + let session = event.session_pk.to_string(); + let stored_at = session_titles::table + .find(&session) + .select(session_titles::title_observed_at) + .first::>(connection) + .optional()? + .flatten(); + if stored_at.is_some_and(|at| at <= event.observed_at) { + return Ok(()); + } + let Some(title) = default_title(event.metadata) else { + return Ok(()); + }; + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(&session), + session_titles::user_id.eq(event.user_id.to_string()), + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk.to_string()), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(Utc::now().timestamp_micros()), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk.to_string()), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(Utc::now().timestamp_micros()), + )) + .execute(connection)?; + Ok(()) +} + fn naming_call( connection: &mut SqliteConnection, event: &CapturedEvent<'_>, @@ -403,7 +450,8 @@ fn match_naming_thread( pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { run_once(connection, BACKFILL_NAME, backfill_titles)?; run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; - run_once(connection, LINK_REPAIR_NAME, backfill_links) + run_once(connection, LINK_REPAIR_NAME, backfill_links)?; + run_once(connection, DEFAULT_BACKFILL_NAME, backfill_defaults) } fn run_once( @@ -518,6 +566,69 @@ fn backfill_links(connection: &mut SqliteConnection) -> Result<(), AppError> { Ok(()) } +/// Visit requests in time order until the first readable user input is found. +fn backfill_defaults(connection: &mut SqliteConnection) -> Result<(), AppError> { + let sessions = agent_sessions::table + .filter(agent_sessions::agent_name.ne_all(["codex", "claude-code"])) + .select(( + agent_sessions::id, + agent_sessions::user_id, + agent_sessions::agent_name, + )) + .load::<(String, String, String)>(connection)?; + for (session, user_id, agent_name) in sessions { + let mut offset = 0_i64; + loop { + let requests = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(&session)) + .filter(llm_usage_events::event_type.eq("request")) + .order_by(( + llm_usage_events::observed_at.asc(), + llm_usage_events::id.asc(), + )) + .offset(offset) + .limit(16) + .select(( + llm_usage_events::id, + llm_usage_events::observed_at, + llm_usage_events::metadata, + )) + .load::<(String, i64, String)>(connection)?; + if requests.is_empty() { + break; + } + let mut found = false; + for (event_pk, observed_at, metadata) in requests { + let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); + if default_title(&metadata).is_none() { + continue; + } + capture_default( + connection, + &CapturedEvent { + user_id: parse_uuid(&user_id, "user id")?, + session_pk: parse_uuid(&session, "session id")?, + event_pk: parse_uuid(&event_pk, "event id")?, + turn_index: 0, + agent_name: &agent_name, + event_type: "request", + text: None, + metadata: &metadata, + observed_at, + }, + )?; + found = true; + break; + } + if found { + break; + } + offset = offset.saturating_add(16); + } + } + Ok(()) +} + /// Adds the stored title, purpose, and matched task session to a session. pub(super) fn fill( connection: &mut SqliteConnection, @@ -718,6 +829,75 @@ mod tests { ); } + #[test] + fn default_titles_keep_the_earliest_user_request_even_when_delivered_late() { + let mut database = database(); + for (id, at, kind, input, expected) in [ + ("r2", "2026-09-30T00:00:02Z", "request", "Second", "Second"), + ("r3", "2026-09-30T00:00:03Z", "request", "Third", "Second"), + ("r1", "2026-09-30T00:00:01Z", "request", "First", "First"), + ( + "s0", + "2026-09-30T00:00:00Z", + "response", + "Response", + "First", + ), + ] { + let mut value = event("cursor", "default", id, at, kind, input); + value["metadata"] = json!({"conversation": {"messages": [ + {"role": "user", "text": input} + ]}}); + ingest(&mut database, &[value]); + assert_eq!( + session_title(&mut database, "default").0.as_deref(), + Some(expected) + ); + } + } + + #[test] + fn backfills_default_titles_past_empty_requests_once() { + let mut database = database(); + let mut values: Vec<_> = (0_i32..17_i32) + .map(|index| { + event( + "cursor", + "default", + &format!("empty-{index}"), + "2026-09-30T00:00:00Z", + "request", + "No structured user input", + ) + }) + .collect(); + let mut first = event( + "cursor", + "default", + "first", + "2026-09-30T00:00:01Z", + "request", + "First request", + ); + archive(&mut first, "user"); + values.push(first); + ingest(&mut database, &values); + diesel::delete(session_titles::table) + .execute(&mut database) + .expect("clear titles"); + + backfill(&mut database).expect("backfill"); + assert_eq!( + session_title(&mut database, "default").0.as_deref(), + Some("First request") + ); + diesel::delete(session_titles::table) + .execute(&mut database) + .expect("clear titles"); + backfill(&mut database).expect("second backfill"); + assert_eq!(session_title(&mut database, "default"), (None, None)); + } + #[test] fn marks_codex_naming_threads_as_auxiliary_even_when_the_response_arrives_first() { let mut database = database(); @@ -860,8 +1040,8 @@ mod tests { .count() .get_result::(&mut database) .expect("count"), - 3, - "titles, initial links and repaired links are each scanned once" + 4, + "generated titles, both link versions and default titles are each scanned once" ); } diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 5059aa9..2ae4afc 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -531,7 +531,7 @@ pub struct SessionInfo { pub ended_at: Option>, /// Stable, allowlisted session metadata. pub metadata: Value, - /// Name the Agent generated for the session, when one was captured. + /// Agent-generated title, or a first-user-input preview for other Agents. pub title: Option, /// Why the session exists when it only serves the Agent, such as /// `title_generation`; `None` for ordinary sessions. diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index 192b593..1e445b3 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -110,7 +110,7 @@ pub struct SessionListItem { pub turns: i64, pub events: i64, pub total_tokens: i64, - /// Name the Agent generated for the session, when one was captured. + /// Agent-generated title, or a first-user-input preview for other Agents. pub title: Option, /// Why the session exists when it only serves the Agent, such as /// `title_generation`; `None` for ordinary sessions. diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index f35cd72..ae02a1c 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -1,4 +1,4 @@ -//! Session names recovered from the Agents' own title-generation calls. +//! Agent-generated session names, or first-user-input previews for other Agents. //! //! Claude Code and Codex ask a model to name each session and record the //! answer as `{"title": "..."}`. Those calls are captured like any other, so @@ -145,6 +145,39 @@ pub fn opens_with_prompt(metadata: &Value, prompt: &str) -> bool { codex_user_message(metadata, "user").is_some_and(|text| text.trim_start().starts_with(prompt)) } +/// Default name: the first non-empty user input, on one line, at most 80 chars. +pub fn default_title(metadata: &Value) -> Option { + let input = if metadata.get("raw_body").is_some() { + let values = bodies(metadata)?; + let payload = request_payload(&values)?; + if let Some(input) = payload.get("input").and_then(Value::as_str) { + input.to_owned() + } else { + request_messages(payload) + .into_iter() + .find(|message| message.role == "user" && !message.text.trim().is_empty())? + .text + } + } else { + metadata + .pointer("/conversation/messages")? + .as_array()? + .iter() + .filter(|message| message["role"] == "user") + .filter_map(|message| message["text"].as_str()) + .find(|text| !text.trim().is_empty())? + .to_owned() + }; + let input = input.split_whitespace().collect::>().join(" "); + let mut chars = input.chars(); + let mut title: String = chars.by_ref().take(80).collect(); + if chars.next().is_some() { + title.pop(); + title.push('…'); + } + (!title.is_empty()).then_some(title) +} + #[derive(Deserialize)] struct TitleObject { title: String, @@ -318,6 +351,54 @@ mod tests { }) } + #[test] + fn default_title_reads_the_first_nonempty_user_input() { + let messages = serde_json::json!([ + {"role": "system", "content": "System instructions"}, + {"role": "assistant", "content": "Assistant output"}, + {"role": "tool", "content": "Tool output"}, + {"role": "user", "content": " "}, + {"role": "user", "content": [{"type": "text", "text": " 修复\n 登录问题 "}]}, + {"role": "user", "content": "Second request"} + ]); + for key in ["input", "messages"] { + let metadata = archived_request(&serde_json::json!({key: messages}), "unused"); + assert_eq!(default_title(&metadata).as_deref(), Some("修复 登录问题")); + } + let legacy = serde_json::json!({"conversation": {"messages": [ + {"role": "system", "text": "Instructions"}, + {"role": "user", "text": "First request"}, + {"role": "user", "text": "Second request"} + ]}}); + assert_eq!(default_title(&legacy).as_deref(), Some("First request")); + } + + #[test] + fn default_title_truncates_unicode_and_leaves_short_input_intact() { + for (input, expected) in [ + ("修复登录".to_owned(), "修复登录".to_owned()), + ("修".repeat(80), "修".repeat(80)), + ("修".repeat(81), format!("{}…", "修".repeat(79))), + ] { + let metadata = archived_request(&serde_json::json!({"input": input}), "unused"); + assert_eq!(default_title(&metadata), Some(expected)); + } + } + + #[test] + fn default_title_requires_readable_user_input() { + for payload in [ + serde_json::json!({"input": " \n "}), + serde_json::json!({"messages": [{"role": "system", "content": "Instructions"}]}), + serde_json::json!({"messages": [{"role": "user", "content": [ + {"type": "tool_result", "content": "Output"} + ]}]}), + ] { + assert_eq!(default_title(&archived_request(&payload, "unused")), None); + } + assert_eq!(default_title(&serde_json::json!({"text": "No role"})), None); + } + #[test] fn other_agents_keep_the_default_session_strategy() { let text = format!("{CLAUDE_CODE_NAMING_INSTRUCTION}\n{CODEX_NAMING_INSTRUCTION}"); @@ -391,6 +472,7 @@ mod tests { let metadata = serde_json::json!({"raw_body": {"parts": [part]}, "conversation": {"messages": [{"role": "user", "text": "hello"}]}}); assert!(!opens_with_prompt(&metadata, "hello")); + assert_eq!(default_title(&metadata), None); assert!( NamingRequest::parse("codex", Some(CODEX_NAMING_INSTRUCTION), &metadata).is_none() ); diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py index 683c7c1..0d34471 100644 --- a/scripts/tests/session_titles_api.py +++ b/scripts/tests/session_titles_api.py @@ -70,8 +70,8 @@ def session(name): for agent, name, instruction, expected in [ ("claude-code", "claude", CLAUDE_NAMING, "Build fix"), - ("cursor", "generic-claude", CLAUDE_NAMING, None), - ("cursor", "generic-codex", CODEX_NAMING, None), + ("cursor", "generic-claude", CLAUDE_NAMING, CLAUDE_NAMING), + ("cursor", "generic-codex", CODEX_NAMING, " ".join(CODEX_NAMING.split())[:79] + "…"), ("codex", "quoted-user", CODEX_NAMING, None), ]: ingest(make(name, "request", instruction, agent=agent), @@ -79,4 +79,23 @@ def session(name): value = session(name) require(value["title"] == expected, f"{agent}/{name} follows its adapter policy") require(value["purpose"] is None, f"{agent}/{name} is not an auxiliary title thread") + + first = make("default", "request", " 修复\n登录问题 " + "好" * 90, agent="cursor") + ingest(first) + expected = "修复 登录问题 " + "好" * 71 + "…" + require(session("default")["title"] == expected, "default title truncates Unicode user input") + later = make("default", "request", "Later request", agent="cursor") + later["observed_at"] = "2026-09-30T00:00:03Z" + ingest(later, make("default", "response", '{"title":"Do not rename"}', agent="cursor")) + require(session("default")["title"] == expected, "later requests and responses preserve the first input") + earlier = make("default", "request", "Earlier request", agent="cursor") + earlier["observed_at"] = "2026-09-30T00:00:00Z" + ingest(earlier) + require(session("default")["title"] == "Earlier request", "late delivery restores the earliest input") + + result = request(base_url, "GET", "/v1/agent-usage/sessions?agent_name=cursor&page_size=100", token=token) + require_status(result, 200, "default title session list") + pk = session("default")["session_pk"] + listed = next(item for item in result.json()["items"] if item["session_pk"] == pk) + require(listed["title"] == "Earlier request", "session list and outline share the default title") print("session title adapter API contract passed") From 26f7eb37e776078e27e2e41678fa0fad8c5c25df Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 02:08:58 -0700 Subject: [PATCH 16/30] feat(filters): match any of several agents, sessions, providers, and models Query endpoints accept repeated agent_name, session_id, llm_provider, and llm_model keys; values within one filter match any, filters still combine. Single-value calls are unchanged. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 61 +++++++- crates/abyss-backend/Cargo.toml | 1 + crates/abyss-backend/src/api.rs | 5 +- crates/abyss-backend/src/search/mod.rs | 58 ++++--- .../src/storage/postgres/repository.rs | 142 +++++++++--------- .../storage/postgres/search/elasticsearch.rs | 29 +++- .../src/storage/postgres/session_list.rs | 26 ++-- .../src/storage/sqlite/repository.rs | 60 ++++---- .../src/storage/sqlite/search.rs | 34 ++--- .../src/storage/sqlite/session_list.rs | 52 ++++--- crates/abyss-backend/src/usage/mod.rs | 40 +++-- crates/abyss-backend/src/usage/persistence.rs | 38 +++++ .../abyss-backend/src/usage/session_list.rs | 53 +++---- scripts/tests/blackbox_api.py | 65 +++++++- scripts/tests/mock_elasticsearch.py | 4 + 15 files changed, 442 insertions(+), 226 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cad5933..38200e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,6 +7,7 @@ name = "abyss-backend" version = "1.0.0" dependencies = [ "axum", + "axum-extra", "base64 0.21.7", "chrono", "diesel", @@ -110,6 +111,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-extra" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be44683b41ccb9ab2d23a5230015c9c3c55be97a25e4428366de8873103f7970" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "serde_core", + "serde_html_form", + "serde_path_to_error", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.21.7" @@ -378,6 +402,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -494,6 +524,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -745,6 +781,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -1185,7 +1231,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", "thiserror", ] @@ -1287,6 +1333,19 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "serde_html_form" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f2d7ff8a2140333718bb329f5c40fc5f0865b84c426183ce14c97d2ab8154f" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "ryu", + "serde_core", +] + [[package]] name = "serde_json" version = "1.0.151" diff --git a/crates/abyss-backend/Cargo.toml b/crates/abyss-backend/Cargo.toml index e926122..89581e7 100644 --- a/crates/abyss-backend/Cargo.toml +++ b/crates/abyss-backend/Cargo.toml @@ -31,6 +31,7 @@ sqlite-fts = [ [dependencies] axum = "0.8" +axum-extra = { version = "0.12", default-features = false, features = ["query"] } base64 = "0.21" chrono = { version = "0.4", features = ["serde"] } diesel = { version = "2.2", default-features = false, features = ["32-column-tables", "r2d2", "with-deprecated"], optional = true } diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index a386557..3f24eb0 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -7,11 +7,14 @@ use axum::{ Json, Router, body::Bytes, - extract::{DefaultBodyLimit, Path, Query, State}, + extract::{DefaultBodyLimit, Path, State}, http::{HeaderMap, HeaderValue, header}, response::{IntoResponse, Response}, routing::{get, post}, }; +// Filter keys repeat to select several values, e.g. `llm_model=a&llm_model=b`; +// axum's own Query cannot collect repeated keys. +use axum_extra::extract::Query; use serde::Serialize; use tower_http::trace::TraceLayer; use uuid::Uuid; diff --git a/crates/abyss-backend/src/search/mod.rs b/crates/abyss-backend/src/search/mod.rs index c2e9164..718d5fc 100644 --- a/crates/abyss-backend/src/search/mod.rs +++ b/crates/abyss-backend/src/search/mod.rs @@ -32,12 +32,15 @@ pub struct SessionSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Optional canonical Agent filter. - pub agent_name: Option, - /// Optional canonical LLM provider filter. - pub llm_provider: Option, - /// Optional model filter. - pub llm_model: Option, + /// Optional canonical Agent filter; repeat the key to match any of several. + #[serde(default)] + pub agent_name: Vec, + /// Optional canonical LLM provider filter; repeat the key to match any of several. + #[serde(default)] + pub llm_provider: Vec, + /// Optional model filter; repeat the key to match any of several. + #[serde(default)] + pub llm_model: Vec, /// Optional request or response filter. pub event_type: Option, /// One-based result page. @@ -93,11 +96,9 @@ impl SessionSearchQuery { text, from: self.from, to: self.to, - agent_name: normalized_optional(self.agent_name.as_deref(), "agent_name")? - .map(|value| canonical_agent_name(&value)), - llm_provider: normalized_optional(self.llm_provider.as_deref(), "llm_provider")? - .map(|value| normalize_slug(&value)), - llm_model: normalized_optional(self.llm_model.as_deref(), "llm_model")?, + agent_names: normalized_list(&self.agent_name, "agent_name", canonical_agent_name)?, + llm_providers: normalized_list(&self.llm_provider, "llm_provider", normalize_slug)?, + llm_models: normalized_list(&self.llm_model, "llm_model", str::to_owned)?, event_type, page, page_size, @@ -114,12 +115,12 @@ pub struct ValidatedSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Canonical Agent-name filter. - pub agent_name: Option, - /// Canonical provider slug filter. - pub llm_provider: Option, - /// Trimmed model filter. - pub llm_model: Option, + /// Canonical Agent names to match any of, or empty for every Agent. + pub agent_names: Vec, + /// Canonical provider slugs to match any of, or empty for every provider. + pub llm_providers: Vec, + /// Trimmed models to match any of, or empty for every model. + pub llm_models: Vec, /// Canonical request or response filter. pub event_type: Option, /// One-based result page. @@ -270,6 +271,23 @@ fn normalized_optional(value: Option<&str>, field: &str) -> Result String, +) -> Result, AppError> { + let mut normalized = values + .iter() + .map(|value| { + normalized_required(value, field, MAX_FILTER_CHARACTERS).map(|value| normalize(&value)) + }) + .collect::, _>>()?; + normalized.sort_unstable(); + normalized.dedup(); + Ok(normalized) +} + fn normalize_slug(value: &str) -> String { value .trim() @@ -338,9 +356,9 @@ mod tests { q: q.to_owned(), from: None, to: None, - agent_name: None, - llm_provider: None, - llm_model: None, + agent_name: Vec::new(), + llm_provider: Vec::new(), + llm_model: Vec::new(), event_type: None, page: None, page_size: None, diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index a378a9b..3494363 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -46,10 +46,11 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, TurnIdentityKind, agent_name_filter_values, canonical_agent_name, - choose_turn_index, escape_like_pattern, non_empty, non_negative_count, normalize_limit, - normalize_slug, normalize_text, normalized_optional, normalized_tokens, parse_group_by, - sha256_bytes, turn_identity, unix_epoch, validate_batch, validate_event_type, + TurnIdentity, TurnIdentityKind, agent_names_filter, canonical_agent_name, + choose_turn_index, escape_like_pattern, exact_values_filter, non_empty, + non_negative_count, normalize_limit, normalize_slug, normalize_text, + normalized_optional, normalized_tokens, parse_group_by, providers_filter, sha256_bytes, + turn_identity, unix_epoch, validate_batch, validate_event_type, }, session_metadata_from_event, }, @@ -103,11 +104,11 @@ const SUMMARY_AGGREGATE_SQL: &str = "\ )) \ ) \ AND ($17 = false OR e.agent_name = ANY($18)) \ - AND ($19 = false OR e.session_id = $20) \ + AND ($19 = false OR e.session_id = ANY($20)) \ AND ($21 = false OR e.session_pk = $22) \ AND ($23 = false OR e.turn_index = $24) \ - AND ($25 = false OR e.llm_provider = $26) \ - AND ($27 = false OR e.llm_model = $28) \ + AND ($25 = false OR e.llm_provider = ANY($26)) \ + AND ($27 = false OR e.llm_model = ANY($28)) \ AND ($29 = false OR e.event_type = $30) \ GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 \ ORDER BY total_tokens DESC, agent_name ASC NULLS LAST, user_email ASC NULLS LAST, day ASC NULLS LAST \ @@ -251,12 +252,12 @@ pub fn usage_summary( from: query.from, to: query.to, user_filter: query.user_filter.as_deref(), - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: None, turn_index: None, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit, offset: 0, @@ -291,16 +292,16 @@ fn load_filtered_summary_rows( let has_user_filter = !user_filter_pattern.is_empty(); let agent_names = sql_filters.agent_names; let has_agent_filter = !agent_names.is_empty(); - let session_id = sql_filters.session_id.unwrap_or_default(); - let has_session_id_filter = !session_id.is_empty(); + let session_ids = sql_filters.session_ids; + let has_session_id_filter = !session_ids.is_empty(); let session_pk = sql_filters.session_pk.unwrap_or_else(Uuid::nil); let has_session_pk_filter = session_pk != Uuid::nil(); let turn_index = sql_filters.turn_index.unwrap_or_default(); let has_turn_index_filter = turn_index > 0_i32; - let provider = sql_filters.provider.unwrap_or_default(); - let has_provider_filter = !provider.is_empty(); - let model = sql_filters.model.unwrap_or_default(); - let has_model_filter = !model.is_empty(); + let providers = sql_filters.providers; + let has_provider_filter = !providers.is_empty(); + let models = sql_filters.models; + let has_model_filter = !models.is_empty(); let event_type = sql_filters.event_type.unwrap_or_default(); let has_event_type_filter = !event_type.is_empty(); @@ -327,15 +328,15 @@ fn load_filtered_summary_rows( .bind::(has_agent_filter) .bind::, _>(agent_names) .bind::(has_session_id_filter) - .bind::(&session_id) + .bind::, _>(session_ids) .bind::(has_session_pk_filter) .bind::(session_pk) .bind::(has_turn_index_filter) .bind::(turn_index) .bind::(has_provider_filter) - .bind::(&provider) + .bind::, _>(providers) .bind::(has_model_filter) - .bind::(&model) + .bind::, _>(models) .bind::(has_event_type_filter) .bind::(&event_type) .bind::(filters.limit) @@ -567,12 +568,12 @@ pub fn raw_events( from: query.from, to: query.to, user_filter: None, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: query.session_pk, turn_index: query.turn_index, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit: page_size.saturating_add(1_i64), offset, @@ -1042,12 +1043,13 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(to)); } - if let Some(agent_name) = filters.agent_name.and_then(non_empty) { - let normalized = agent_name_filter_values(agent_name); - query = query.filter(llm_usage_events::agent_name.eq_any(normalized)); + let agent_names = agent_names_filter(filters.agent_names); + if !agent_names.is_empty() { + query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } - if let Some(session_id) = filters.session_id.and_then(non_empty) { - query = query.filter(llm_usage_events::session_id.eq(session_id)); + let session_ids = exact_values_filter(filters.session_ids); + if !session_ids.is_empty() { + query = query.filter(llm_usage_events::session_id.eq_any(session_ids)); } if let Some(session_pk) = filters.session_pk { query = query.filter(llm_usage_events::session_pk.eq(session_pk)); @@ -1055,12 +1057,13 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - if let Some(provider) = filters.llm_provider.and_then(non_empty) { - let normalized = normalize_slug(provider); - query = query.filter(llm_usage_events::llm_provider.eq(normalized)); + let providers = providers_filter(filters.llm_providers); + if !providers.is_empty() { + query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } - if let Some(model) = filters.llm_model.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_model.eq(model.trim())); + let models = exact_values_filter(filters.llm_models); + if !models.is_empty() { + query = query.filter(llm_usage_events::llm_model.eq_any(models)); } if let Some(event_type) = event_type { query = query.filter(llm_usage_events::event_type.eq(event_type)); @@ -1215,12 +1218,13 @@ struct EventFilters<'a> { from: Option>, to: Option>, user_filter: Option<&'a str>, - agent_name: Option<&'a str>, - session_id: Option<&'a str>, + /// Each list matches any of its values; an empty list matches everything. + agent_names: &'a [String], + session_ids: &'a [String], session_pk: Option, turn_index: Option, - llm_provider: Option<&'a str>, - llm_model: Option<&'a str>, + llm_providers: &'a [String], + llm_models: &'a [String], event_type: Option<&'a str>, limit: i64, offset: i64, @@ -1281,11 +1285,11 @@ struct SummarySqlFilters { user_filter_id: Option, user_filter_pattern: Option, agent_names: Vec, - session_id: Option, + session_ids: Vec, session_pk: Option, turn_index: Option, - provider: Option, - model: Option, + providers: Vec, + models: Vec, event_type: Option, } @@ -1295,20 +1299,10 @@ impl SummarySqlFilters { let user_filter_id = user_filter.and_then(|value| Uuid::parse_str(value).ok()); let user_filter_pattern = user_filter.map(|value| format!("%{}%", escape_like_pattern(value))); - let agent_names = filters - .agent_name - .and_then(non_empty) - .map(agent_name_filter_values) - .unwrap_or_default(); - let session_id = filters - .session_id - .and_then(non_empty) - .map(|value| value.trim().to_owned()); - let provider = filters.llm_provider.and_then(non_empty).map(normalize_slug); - let model = filters - .llm_model - .and_then(non_empty) - .map(|value| value.trim().to_owned()); + let agent_names = agent_names_filter(filters.agent_names); + let session_ids = exact_values_filter(filters.session_ids); + let providers = providers_filter(filters.llm_providers); + let models = exact_values_filter(filters.llm_models); let event_type = filters.event_type.map(validate_event_type).transpose()?; Ok(Self { @@ -1318,11 +1312,11 @@ impl SummarySqlFilters { user_filter_id, user_filter_pattern, agent_names, - session_id, + session_ids, session_pk: filters.session_pk, turn_index: filters.turn_index, - provider, - model, + providers, + models, event_type, }) } @@ -1424,9 +1418,9 @@ mod tests { use super::{ EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, TurnIdentityKind, - agent_name_filter_values, canonical_agent_name, choose_turn_index, event_response, - latest_optional, normalize_limit, normalize_slug, normalized_tokens, parse_group_by, - turn_identity_from_metadata, validate_batch, validate_event, validate_event_type, + canonical_agent_name, choose_turn_index, event_response, latest_optional, normalize_limit, + normalize_slug, normalized_tokens, parse_group_by, turn_identity_from_metadata, + validate_batch, validate_event, validate_event_type, }; use chrono::{DateTime, TimeZone, Utc}; use serde_json::json; @@ -1437,7 +1431,7 @@ mod tests { error::AppError, usage::{ AgentPayload, DevicePayload, IngestEventsRequest, IngestUsageEvent, LlmPayload, - SummaryRow, TokenUsagePayload, UsageEventType, + SummaryRow, TokenUsagePayload, UsageEventType, persistence::agent_name_filter_values, }, }; @@ -1676,12 +1670,16 @@ mod tests { from: Some(timestamp(2_u32, 0_u32, 0_u32)), to: None, user_filter: Some(" Alice_% "), - agent_name: Some("Claude Desktop"), - session_id: Some(" session-1 "), + agent_names: &["Claude Desktop".to_owned(), "codex".to_owned()], + session_ids: &[" session-1 ".to_owned(), "session-2".to_owned()], session_pk: None, turn_index: None, - llm_provider: Some("OpenAI"), - llm_model: Some(" gpt-5.5 "), + llm_providers: &["OpenAI".to_owned(), " ".to_owned()], + llm_models: &[ + " gpt-5.5 ".to_owned(), + "gpt-5.5".to_owned(), + "o3".to_owned(), + ], event_type: Some(" Response "), limit: 50_i64, offset: 0_i64, @@ -1700,17 +1698,13 @@ mod tests { ); assert_eq!( sql_filters.agent_names, - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] + ["claude", "claude-code", "claude-desktop", "codex"] ); - assert_eq!(sql_filters.session_id.as_deref(), Some("session-1")); + assert_eq!(sql_filters.session_ids, ["session-1", "session-2"]); assert_eq!(sql_filters.session_pk, None); assert_eq!(sql_filters.turn_index, None); - assert_eq!(sql_filters.provider.as_deref(), Some("openai")); - assert_eq!(sql_filters.model.as_deref(), Some("gpt-5.5")); + assert_eq!(sql_filters.providers, ["openai"]); + assert_eq!(sql_filters.models, ["gpt-5.5", "o3"]); assert_eq!(sql_filters.event_type.as_deref(), Some("response")); } diff --git a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs index ee59dfa..578e328 100644 --- a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs +++ b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs @@ -322,9 +322,9 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { } filters.push(json!({"range": {"observed_at": Value::Object(range)}})); } - push_term_filter(&mut filters, "agent_name", query.agent_name.as_deref()); - push_term_filter(&mut filters, "llm_provider", query.llm_provider.as_deref()); - push_term_filter(&mut filters, "llm_model", query.llm_model.as_deref()); + push_terms_filter(&mut filters, "agent_name", &query.agent_names); + push_terms_filter(&mut filters, "llm_provider", &query.llm_providers); + push_terms_filter(&mut filters, "llm_model", &query.llm_models); push_term_filter(&mut filters, "event_type", query.event_type.as_deref()); json!({ @@ -402,6 +402,15 @@ fn push_term_filter(filters: &mut Vec, field: &str, value: Option<&str>) } } +/// Matches any of `values`; an empty list adds no filter. +fn push_terms_filter(filters: &mut Vec, field: &str, values: &[String]) { + if !values.is_empty() { + let mut terms = serde_json::Map::new(); + terms.insert(field.to_owned(), json!(values)); + filters.push(json!({"terms": Value::Object(terms)})); + } +} + fn append_json_line( body: &mut String, value: &T, @@ -625,9 +634,9 @@ mod tests { text: "timeout".to_owned(), from: Some(Utc.with_ymd_and_hms(2026, 8, 1, 0, 0, 0).unwrap()), to: None, - agent_name: Some("claude-code".to_owned()), - llm_provider: None, - llm_model: None, + agent_names: vec!["claude-code".to_owned(), "codex".to_owned()], + llm_providers: Vec::new(), + llm_models: vec!["gpt-5".to_owned()], event_type: None, page: 1, page_size: 20, @@ -640,7 +649,13 @@ mod tests { .as_array() .expect("filters should be an array"); assert!(filters.contains(&json!({"term": {"user_id": user_id}}))); - assert!(filters.contains(&json!({"term": {"agent_name": "claude-code"}}))); + assert!(filters.contains(&json!({"terms": {"agent_name": ["claude-code", "codex"]}}))); + assert!(filters.contains(&json!({"terms": {"llm_model": ["gpt-5"]}}))); + assert!( + !filters + .iter() + .any(|filter| filter["terms"].get("llm_provider").is_some()) + ); assert_eq!(request["collapse"]["field"], "session_pk"); } diff --git a/crates/abyss-backend/src/storage/postgres/session_list.rs b/crates/abyss-backend/src/storage/postgres/session_list.rs index e89ec06..112ef4d 100644 --- a/crates/abyss-backend/src/storage/postgres/session_list.rs +++ b/crates/abyss-backend/src/storage/postgres/session_list.rs @@ -15,8 +15,8 @@ use crate::{ /// Per-session activity with the filters that apply before paging. /// /// Bind order: `$1` owner, `$2`/`$3` agent filter flag and names, `$4`/`$5` -/// lower activity bound, `$6`/`$7` upper start bound, `$8`/`$9` provider and -/// `$10`/`$11` model a session must have used. +/// lower activity bound, `$6`/`$7` upper start bound, `$8`/`$9` providers and +/// `$10`/`$11` models a session must have used at least one of. const ACTIVITY_CTE: &str = "\ WITH activity AS ( \ SELECT \ @@ -42,10 +42,10 @@ const ACTIVITY_CTE: &str = "\ AND ($2 = false OR s.agent_name = ANY($3)) \ AND ($8 = false OR EXISTS ( \ SELECT 1 FROM llm_usage_events p \ - WHERE p.session_pk = s.id AND p.llm_provider = $9)) \ + WHERE p.session_pk = s.id AND p.llm_provider = ANY($9))) \ AND ($10 = false OR EXISTS ( \ SELECT 1 FROM llm_usage_events m \ - WHERE m.session_pk = s.id AND m.llm_model = $11)) \ + WHERE m.session_pk = s.id AND m.llm_model = ANY($11))) \ GROUP BY s.id, d.host_name, d.platform, st.title, st.purpose \ ), \ filtered AS ( \ @@ -100,8 +100,6 @@ pub(super) fn list_sessions( let has_agents = !query.agent_names.is_empty(); let from = query.from.unwrap_or(DateTime::UNIX_EPOCH); let to = query.to.unwrap_or(DateTime::UNIX_EPOCH); - let provider = query.llm_provider.as_deref().unwrap_or_default(); - let model = query.llm_model.as_deref().unwrap_or_default(); let total = sql_query(format!( "{ACTIVITY_CTE} SELECT COUNT(*)::bigint AS total FROM filtered" @@ -113,10 +111,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) - .bind::(query.llm_provider.is_some()) - .bind::(provider) - .bind::(query.llm_model.is_some()) - .bind::(model) + .bind::(!query.llm_providers.is_empty()) + .bind::, _>(&query.llm_providers) + .bind::(!query.llm_models.is_empty()) + .bind::, _>(&query.llm_models) .get_result::(connection)? .total; @@ -131,10 +129,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) - .bind::(query.llm_provider.is_some()) - .bind::(provider) - .bind::(query.llm_model.is_some()) - .bind::(model) + .bind::(!query.llm_providers.is_empty()) + .bind::, _>(&query.llm_providers) + .bind::(!query.llm_models.is_empty()) + .bind::, _>(&query.llm_models) .bind::(i64::from(query.page_size)) .bind::(query.offset) .load::(connection)?; diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index ac7d060..3817e98 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -27,10 +27,10 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, agent_name_filter_values, canonical_agent_name, choose_turn_index, - non_empty, normalize_limit, normalize_slug, normalize_text, normalized_optional, - normalized_tokens, parse_group_by, sha256_bytes, turn_identity, validate_batch, - validate_event_type, + TurnIdentity, agent_names_filter, canonical_agent_name, choose_turn_index, + exact_values_filter, non_empty, normalize_limit, normalize_slug, normalize_text, + normalized_optional, normalized_tokens, parse_group_by, providers_filter, sha256_bytes, + turn_identity, validate_batch, validate_event_type, }, session_metadata_from_event, }, @@ -542,12 +542,12 @@ pub(super) fn raw_events( user_id, from: query.from, to: query.to, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: query.session_pk, turn_index: query.turn_index, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit: page_size.saturating_add(1), offset, @@ -581,12 +581,13 @@ struct EventFilters<'a> { user_id: Uuid, from: Option>, to: Option>, - agent_name: Option<&'a str>, - session_id: Option<&'a str>, + /// Each list matches any of its values; an empty list matches everything. + agent_names: &'a [String], + session_ids: &'a [String], session_pk: Option, turn_index: Option, - llm_provider: Option<&'a str>, - llm_model: Option<&'a str>, + llm_providers: &'a [String], + llm_models: &'a [String], event_type: Option<&'a str>, limit: i64, offset: i64, @@ -605,12 +606,13 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(timestamp_to_micros(to))); } - if let Some(agent_name) = filters.agent_name.and_then(non_empty) { - query = - query.filter(llm_usage_events::agent_name.eq_any(agent_name_filter_values(agent_name))); + let agent_names = agent_names_filter(filters.agent_names); + if !agent_names.is_empty() { + query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } - if let Some(session_id) = filters.session_id.and_then(non_empty) { - query = query.filter(llm_usage_events::session_id.eq(session_id.trim())); + let session_ids = exact_values_filter(filters.session_ids); + if !session_ids.is_empty() { + query = query.filter(llm_usage_events::session_id.eq_any(session_ids)); } if let Some(session_pk) = filters.session_pk { query = query.filter(llm_usage_events::session_pk.eq(session_pk.to_string())); @@ -618,11 +620,13 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - if let Some(provider) = filters.llm_provider.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_provider.eq(normalize_slug(provider))); + let providers = providers_filter(filters.llm_providers); + if !providers.is_empty() { + query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } - if let Some(model) = filters.llm_model.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_model.eq(model.trim())); + let models = exact_values_filter(filters.llm_models); + if !models.is_empty() { + query = query.filter(llm_usage_events::llm_model.eq_any(models)); } if let Some(event_type) = filters.event_type.and_then(non_empty) { query = query.filter(llm_usage_events::event_type.eq(validate_event_type(event_type)?)); @@ -960,12 +964,12 @@ pub(super) fn usage_summary( user_id, from: query.from, to: query.to, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: None, turn_index: None, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit: scan_limit, offset: 0, @@ -1377,9 +1381,9 @@ mod tests { q: "searchable response".to_owned(), from: None, to: None, - agent_name: Some("codex".to_owned()), - llm_provider: None, - llm_model: None, + agent_name: vec!["claude-code".to_owned(), "codex".to_owned()], + llm_provider: Vec::new(), + llm_model: Vec::new(), event_type: None, page: None, page_size: None, diff --git a/crates/abyss-backend/src/storage/sqlite/search.rs b/crates/abyss-backend/src/storage/sqlite/search.rs index bdd4ead..f5d1e56 100644 --- a/crates/abyss-backend/src/storage/sqlite/search.rs +++ b/crates/abyss-backend/src/storage/sqlite/search.rs @@ -121,24 +121,9 @@ impl MatchSource { sql.push_str(" AND e.observed_at < ?"); binds.push(FtsBind::BigInt(timestamp_to_micros(to))); } - push_exact_filter( - &mut sql, - &mut binds, - "e.agent_name", - query.agent_name.as_deref(), - ); - push_exact_filter( - &mut sql, - &mut binds, - "e.llm_provider", - query.llm_provider.as_deref(), - ); - push_exact_filter( - &mut sql, - &mut binds, - "e.llm_model", - query.llm_model.as_deref(), - ); + push_any_filter(&mut sql, &mut binds, "e.agent_name", &query.agent_names); + push_any_filter(&mut sql, &mut binds, "e.llm_provider", &query.llm_providers); + push_any_filter(&mut sql, &mut binds, "e.llm_model", &query.llm_models); push_exact_filter( &mut sql, &mut binds, @@ -163,6 +148,19 @@ fn push_exact_filter( } } +/// Matches any of `values`; an empty list adds no filter. +fn push_any_filter(sql: &mut String, binds: &mut Vec, column: &str, values: &[String]) { + if values.is_empty() { + return; + } + sql.push_str(" AND "); + sql.push_str(column); + sql.push_str(" IN ("); + sql.push_str(&vec!["?"; values.len()].join(", ")); + sql.push(')'); + binds.extend(values.iter().cloned().map(FtsBind::Text)); +} + fn bound_query(sql: String, binds: Vec) -> BoxedSqlQuery<'static, Sqlite, SqlQuery> { let mut query = sql_query(sql).into_boxed::(); for bind in binds { diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index 095f5db..d0adcfe 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -20,7 +20,8 @@ use super::{ /// /// Bind order: `?1` owner, `?2`/`?3` agent filter flag and JSON name list, /// `?4`/`?5` lower activity bound, `?6`/`?7` upper start bound, `?8`/`?9` -/// provider and `?10`/`?11` model a session must have used. +/// JSON provider list and `?10`/`?11` JSON model list a session must have used +/// at least one of. const ACTIVITY_CTE: &str = "\ WITH activity AS ( \ SELECT \ @@ -46,10 +47,12 @@ const ACTIVITY_CTE: &str = "\ AND (?2 = 0 OR s.agent_name IN (SELECT value FROM json_each(?3))) \ AND (?8 = 0 OR EXISTS ( \ SELECT 1 FROM llm_usage_events p \ - WHERE p.session_pk = s.id AND p.llm_provider = ?9)) \ + WHERE p.session_pk = s.id \ + AND p.llm_provider IN (SELECT value FROM json_each(?9)))) \ AND (?10 = 0 OR EXISTS ( \ SELECT 1 FROM llm_usage_events m \ - WHERE m.session_pk = s.id AND m.llm_model = ?11)) \ + WHERE m.session_pk = s.id \ + AND m.llm_model IN (SELECT value FROM json_each(?11)))) \ GROUP BY s.id \ ), \ filtered AS ( \ @@ -101,13 +104,12 @@ pub(super) fn list_sessions( user_id: Uuid, query: &ValidatedSessionListQuery, ) -> Result { - let agent_names = serde_json::to_string(&query.agent_names) - .map_err(|error| AppError::internal(format!("serialize agent filter: {error}")))?; + let agent_names = json_list(&query.agent_names, "agent")?; + let providers = json_list(&query.llm_providers, "provider")?; + let models = json_list(&query.llm_models, "model")?; let has_agents = !query.agent_names.is_empty(); let from = query.from.map_or(0, timestamp_to_micros); let to = query.to.map_or(0, timestamp_to_micros); - let provider = query.llm_provider.as_deref().unwrap_or_default(); - let model = query.llm_model.as_deref().unwrap_or_default(); let total = sql_query(format!( "{ACTIVITY_CTE} SELECT COUNT(*) AS total FROM filtered" @@ -119,10 +121,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) - .bind::(query.llm_provider.is_some()) - .bind::(provider) - .bind::(query.llm_model.is_some()) - .bind::(model) + .bind::(!query.llm_providers.is_empty()) + .bind::(&providers) + .bind::(!query.llm_models.is_empty()) + .bind::(&models) .get_result::(connection)? .total; @@ -137,10 +139,10 @@ pub(super) fn list_sessions( .bind::(from) .bind::(query.to.is_some()) .bind::(to) - .bind::(query.llm_provider.is_some()) - .bind::(provider) - .bind::(query.llm_model.is_some()) - .bind::(model) + .bind::(!query.llm_providers.is_empty()) + .bind::(&providers) + .bind::(!query.llm_models.is_empty()) + .bind::(&models) .bind::(i64::from(query.page_size)) .bind::(query.offset) .load::(connection)?; @@ -169,6 +171,11 @@ pub(super) fn list_sessions( Ok(SessionListResponse::new(query, total, items)) } +fn json_list(values: &[String], filter: &str) -> Result { + serde_json::to_string(values) + .map_err(|error| AppError::internal(format!("serialize {filter} filter: {error}"))) +} + #[cfg(test)] mod tests { use chrono::{TimeZone, Utc}; @@ -268,10 +275,15 @@ mod tests { fn filters_by_agent_alias_and_activity_window() { let mut database = seeded(); let claude = SessionListQuery { - agent_name: Some("Claude Code".to_owned()), + agent_name: vec!["Claude Code".to_owned()], ..SessionListQuery::default() }; assert_eq!(list(&mut database, claude), [("beta".to_owned(), 1, 1)]); + let either = SessionListQuery { + agent_name: vec!["Claude Code".to_owned(), "codex".to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, either).len(), 3); // gamma went quiet before the window; alpha is still active inside it. let window = SessionListQuery { @@ -287,10 +299,10 @@ mod tests { } #[test] - fn filters_by_a_model_or_provider_the_session_used() { + fn filters_by_models_or_providers_the_session_used() { let mut database = seeded(); let mini = SessionListQuery { - llm_model: Some("gpt-5-mini".to_owned()), + llm_model: vec!["gpt-5-mini".to_owned(), "unused-model".to_owned()], ..SessionListQuery::default() }; let ids: Vec<_> = list(&mut database, mini) @@ -300,13 +312,13 @@ mod tests { assert_eq!(ids, ["alpha"]); let openai = SessionListQuery { - llm_provider: Some("OpenAI".to_owned()), + llm_provider: vec!["anthropic".to_owned(), "OpenAI".to_owned()], ..SessionListQuery::default() }; assert_eq!(list(&mut database, openai).len(), 3); let other = SessionListQuery { - llm_provider: Some("anthropic".to_owned()), + llm_provider: vec!["anthropic".to_owned()], ..SessionListQuery::default() }; assert!(list(&mut database, other).is_empty()); diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 1c70ee4..029cd50 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -171,14 +171,18 @@ pub struct SummaryQuery { pub fields: Option, /// User UUID, name, or email filter retained for the summary contract. pub user_filter: Option, - /// Agent-name filter. - pub agent_name: Option, - /// Agent-native session identifier filter. - pub session_id: Option, - /// LLM provider filter. - pub llm_provider: Option, - /// LLM model filter. - pub llm_model: Option, + /// Agent-name filter; repeat the key to match any of several Agents. + #[serde(default)] + pub agent_name: Vec, + /// Agent-native session identifier filter; repeat the key to match any of several. + #[serde(default)] + pub session_id: Vec, + /// LLM provider filter; repeat the key to match any of several providers. + #[serde(default)] + pub llm_provider: Vec, + /// LLM model filter; repeat the key to match any of several models. + #[serde(default)] + pub llm_model: Vec, /// Event-side filter. pub event_type: Option, /// Requested aggregate row limit. @@ -338,18 +342,22 @@ pub struct RawEventsQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Agent-name filter. - pub agent_name: Option, - /// Agent-native session identifier filter. - pub session_id: Option, + /// Agent-name filter; repeat the key to match any of several Agents. + #[serde(default)] + pub agent_name: Vec, + /// Agent-native session identifier filter; repeat the key to match any of several. + #[serde(default)] + pub session_id: Vec, /// Backend session primary-key filter. pub session_pk: Option, /// Normalized turn-number filter. pub turn_index: Option, - /// LLM provider filter. - pub llm_provider: Option, - /// LLM model filter. - pub llm_model: Option, + /// LLM provider filter; repeat the key to match any of several providers. + #[serde(default)] + pub llm_provider: Vec, + /// LLM model filter; repeat the key to match any of several models. + #[serde(default)] + pub llm_model: Vec, /// Request or response filter. pub event_type: Option, /// Requested page size. diff --git a/crates/abyss-backend/src/usage/persistence.rs b/crates/abyss-backend/src/usage/persistence.rs index 9497eb0..0e32d6d 100644 --- a/crates/abyss-backend/src/usage/persistence.rs +++ b/crates/abyss-backend/src/usage/persistence.rs @@ -212,6 +212,44 @@ pub fn agent_name_filter_values(value: &str) -> Vec { } } +/// Stored Agent names matching any requested Agent; empty matches every Agent. +pub fn agent_names_filter(values: &[String]) -> Vec { + distinct( + values + .iter() + .filter_map(|value| non_empty(value)) + .flat_map(agent_name_filter_values), + ) +} + +/// Provider slugs matching any requested provider; empty matches every provider. +pub fn providers_filter(values: &[String]) -> Vec { + distinct( + values + .iter() + .filter_map(|value| non_empty(value)) + .map(normalize_slug), + ) +} + +/// Trimmed values compared exactly, such as models or session identifiers; +/// empty matches every value. +pub fn exact_values_filter(values: &[String]) -> Vec { + distinct( + values + .iter() + .filter_map(|value| non_empty(value)) + .map(normalize_text), + ) +} + +fn distinct(values: impl Iterator) -> Vec { + let mut values: Vec = values.collect(); + values.sort_unstable(); + values.dedup(); + values +} + pub fn normalize_text(value: &str) -> String { value.trim().to_owned() } diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index 192b593..c1bc3f2 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::{ error::AppError, - usage::persistence::{agent_name_filter_values, normalize_slug, normalized_optional}, + usage::persistence::{agent_names_filter, exact_values_filter, providers_filter}, }; const DEFAULT_PAGE_SIZE: u32 = 20; @@ -19,12 +19,16 @@ pub struct SessionListQuery { pub from: Option>, /// Keeps sessions that started before this instant. pub to: Option>, - /// Agent-name filter; Claude aliases match together. - pub agent_name: Option, - /// Keeps sessions with at least one event from this LLM provider. - pub llm_provider: Option, - /// Keeps sessions with at least one event from this LLM model. - pub llm_model: Option, + /// Agent-name filter; repeat the key for several Agents. Claude aliases + /// match together. + #[serde(default)] + pub agent_name: Vec, + /// Keeps sessions with at least one event from any of these LLM providers. + #[serde(default)] + pub llm_provider: Vec, + /// Keeps sessions with at least one event from any of these LLM models. + #[serde(default)] + pub llm_model: Vec, /// 1-based page number. pub page: Option, /// Sessions per page, at most 100. @@ -38,10 +42,10 @@ pub struct ValidatedSessionListQuery { pub to: Option>, /// Stored agent names to match, or empty for every Agent. pub agent_names: Vec, - /// Normalized provider slug a session must have used. - pub llm_provider: Option, - /// Exact model a session must have used. - pub llm_model: Option, + /// Normalized provider slugs; a session must have used one, or empty for any. + pub llm_providers: Vec, + /// Exact models; a session must have used one, or empty for any. + pub llm_models: Vec, pub page: u32, pub page_size: u32, pub offset: i64, @@ -70,12 +74,9 @@ impl SessionListQuery { Ok(ValidatedSessionListQuery { from: self.from, to: self.to, - agent_names: normalized_optional(self.agent_name.as_deref()) - .map(|value| agent_name_filter_values(&value)) - .unwrap_or_default(), - llm_provider: normalized_optional(self.llm_provider.as_deref()) - .map(|value| normalize_slug(&value)), - llm_model: normalized_optional(self.llm_model.as_deref()), + agent_names: agent_names_filter(&self.agent_name), + llm_providers: providers_filter(&self.llm_provider), + llm_models: exact_values_filter(&self.llm_model), page, page_size, offset, @@ -152,7 +153,7 @@ mod tests { #[test] fn expands_claude_aliases_and_computes_the_offset() { let query = SessionListQuery { - agent_name: Some(" Claude ".to_owned()), + agent_name: vec![" Claude ".to_owned(), "codex".to_owned()], page: Some(3), page_size: Some(25), ..SessionListQuery::default() @@ -162,28 +163,28 @@ mod tests { assert_eq!(query.offset, 50); assert_eq!( query.agent_names, - ["claude-code", "claude", "claude-desktop"] + ["claude", "claude-code", "claude-desktop", "codex"] ); } #[test] - fn normalizes_provider_and_keeps_the_model_verbatim() { + fn normalizes_providers_and_keeps_models_verbatim() { let query = SessionListQuery { - llm_provider: Some(" OpenAI ".to_owned()), - llm_model: Some(" gpt-5 ".to_owned()), + llm_provider: vec![" OpenAI ".to_owned(), "anthropic".to_owned()], + llm_model: vec![" gpt-5 ".to_owned(), "Claude-Opus".to_owned()], ..SessionListQuery::default() } .validate() .expect("valid"); - assert_eq!(query.llm_provider.as_deref(), Some("openai")); - assert_eq!(query.llm_model.as_deref(), Some("gpt-5")); + assert_eq!(query.llm_providers, ["anthropic", "openai"]); + assert_eq!(query.llm_models, ["Claude-Opus", "gpt-5"]); let blank = SessionListQuery { - llm_model: Some(" ".to_owned()), + llm_model: vec![" ".to_owned()], ..SessionListQuery::default() } .validate() .expect("valid"); - assert!(blank.llm_model.is_none()); + assert!(blank.llm_models.is_empty()); } #[test] diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index a55bf85..0c585bb 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -253,6 +253,63 @@ def run(base_url: str, token: str, run_id: str) -> None: require(summary["requests"] == 1 and summary["responses"] == 1, "summary event counts are incorrect") require(summary["total_tokens"] == 40, "summary token total is incorrect") + # Repeating a filter key matches any of its values; filters still combine + # with each other. + other_session_id = f"{session_id}-absent" + any_of_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [ + ("session_id", other_session_id), + ("session_id", session_id), + ("agent_name", "claude-code"), + ("agent_name", "Codex"), + ("llm_provider", "anthropic"), + ("llm_provider", "OpenAI"), + ("llm_model", "gpt-5.5"), + ("llm_model", "absent-model"), + ("limit", 10), + ] + ) + any_of_response = request(base_url, "GET", any_of_path, token=token) + require_status(any_of_response, 200, "raw event query with repeated filters") + require( + len(any_of_response.json()["events"]) == 2, + "repeated filter values must match any of the values", + ) + none_of_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("llm_model", "absent-model"), ("llm_model", "other-model")] + ) + none_of_response = request(base_url, "GET", none_of_path, token=token) + require_status(none_of_response, 200, "raw event query with unmatched models") + require( + none_of_response.json()["events"] == [], + "filters must still combine across dimensions", + ) + any_of_summary_path = "/v1/agent-usage/summary?" + urllib.parse.urlencode( + [ + ("group_by", "model"), + ("session_id", session_id), + ("session_id", other_session_id), + ("llm_model", "absent-model"), + ("llm_model", "gpt-5.5"), + ] + ) + any_of_summary = request(base_url, "GET", any_of_summary_path, token=token) + require_status(any_of_summary, 200, "usage summary with repeated filters") + any_of_rows = any_of_summary.json()["rows"] + require( + [(row["llm_model"], row["total_tokens"]) for row in any_of_rows] == [("gpt-5.5", 40)], + "summary must aggregate every repeated filter value", + ) + any_of_sessions_path = "/v1/agent-usage/sessions?" + urllib.parse.urlencode( + [("llm_model", "absent-model"), ("llm_model", "gpt-5.5"), ("page_size", 100)] + ) + any_of_sessions = request(base_url, "GET", any_of_sessions_path, token=token) + require_status(any_of_sessions, 200, "session list with repeated filters") + require( + session_id in {item["session_id"] for item in any_of_sessions.json()["items"]}, + "session list must keep sessions that used any selected model", + ) + session_pk = raw_events[0]["session_pk"] timeline = request( base_url, @@ -349,7 +406,13 @@ def run(base_url: str, token: str, run_id: str) -> None: require_status(unauthenticated_event, 401, "unauthenticated single event query") search_path = "/v1/agent-usage/search?" + urllib.parse.urlencode( - {"q": "blackbox response", "agent_name": "codex"} + [ + ("q", "blackbox response"), + ("agent_name", "claude-code"), + ("agent_name", "codex"), + ("llm_model", "absent-model"), + ("llm_model", "gpt-5.5"), + ] ) search_result: dict[str, Any] = {} for _attempt in range(100): diff --git a/scripts/tests/mock_elasticsearch.py b/scripts/tests/mock_elasticsearch.py index a46fad9..a191690 100755 --- a/scripts/tests/mock_elasticsearch.py +++ b/scripts/tests/mock_elasticsearch.py @@ -149,6 +149,10 @@ def matches_filters(document: dict[str, Any], filters: list[dict[str, Any]]) -> for field, expected in clause["term"].items(): if document.get(field) != expected: return False + if "terms" in clause: + for field, accepted in clause["terms"].items(): + if document.get(field) not in accepted: + return False if "range" in clause: for field, bounds in clause["range"].items(): value = document.get(field) From c6f398083fec9715e989d5479ff473cda9e53955 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 02:11:52 -0700 Subject: [PATCH 17/30] perf(sqlite): aggregate usage summaries in SQL The SQLite summary loaded every matching event row, including text and metadata that hold whole request bodies, parsed the metadata JSON, and grouped in memory. A week of usage meant reading over 2 GB to add up token counts. It now groups in one query that mirrors the PostgreSQL aggregate, reading only the token, grouping, session, and turn columns, with the time bounds always bound so the owner-time index limits the scan. Summaries now cover every event in range, as on PostgreSQL, instead of only the newest 100,000. Co-Authored-By: Claude Opus 5.5 --- .../src/storage/sqlite/models.rs | 9 +- .../src/storage/sqlite/repository.rs | 480 ++++++++++++------ 2 files changed, 324 insertions(+), 165 deletions(-) diff --git a/crates/abyss-backend/src/storage/sqlite/models.rs b/crates/abyss-backend/src/storage/sqlite/models.rs index ca70f21..e55bbe8 100644 --- a/crates/abyss-backend/src/storage/sqlite/models.rs +++ b/crates/abyss-backend/src/storage/sqlite/models.rs @@ -215,7 +215,6 @@ pub(super) struct DeviceRecord { pub(super) id: Uuid, pub(super) host_name: String, pub(super) platform: String, - pub(super) os_version: Option, } pub(super) struct SessionRecord { @@ -298,12 +297,16 @@ pub(super) struct AttachmentRecord { impl DeviceRow { pub(super) fn into_record(self) -> Result { - let _ = (&self.user_id, self.created_at, self.updated_at); + let _ = ( + &self.user_id, + &self.os_version, + self.created_at, + self.updated_at, + ); Ok(DeviceRecord { id: parse_uuid(&self.id, "device id")?, host_name: self.host_name, platform: self.platform, - os_version: self.os_version, }) } } diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index ac7d060..0af3f4a 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper, SqliteConnection, + ExpressionMethods, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, SqliteConnection, dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, @@ -946,7 +946,7 @@ pub(super) fn usage_summary( connection: &mut SqliteConnection, query: &SummaryQuery, user_id: Uuid, - scan_limit: i64, + default_limit: i64, ) -> Result { let group_by = parse_group_by(query.group_by.as_deref()); let dimensions = SummaryDimensions::from_group_by(&group_by); @@ -954,45 +954,65 @@ pub(super) fn usage_summary( if !owner_matches_filter(user_id, &owner, query.user_filter.as_deref()) { return Ok(empty_summary(query, group_by)); } - let events = load_filtered_events( - connection, - &EventFilters { - user_id, - from: query.from, - to: query.to, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), - session_pk: None, - turn_index: None, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), - event_type: query.event_type.as_deref(), - limit: scan_limit, - offset: 0, - }, - )?; - let devices = load_device_contexts(connection, &events)?; - let mut buckets: HashMap = HashMap::new(); - for event in events { - let device = devices.get(&event.device_context_id); - let key = SummaryKey::new(&event, device, user_id, &owner, &dimensions); - buckets.entry(key).or_default().add(&event); - } - let mut rows = buckets + let agent_names = query + .agent_name + .as_deref() + .and_then(non_empty) + .map(agent_name_filter_values) + .unwrap_or_default(); + let agent_names_json = serde_json::to_string(&agent_names) + .map_err(|error| AppError::internal(format!("serialize agent filter: {error}")))?; + let session_id = query + .session_id + .as_deref() + .and_then(non_empty) + .map(str::trim); + let provider = query + .llm_provider + .as_deref() + .and_then(non_empty) + .map(normalize_slug); + let model = query + .llm_model + .as_deref() + .and_then(non_empty) + .map(str::trim); + let event_type = query + .event_type + .as_deref() + .and_then(non_empty) + .map(validate_event_type) + .transpose()?; + let limit = normalize_limit(query.limit, default_limit, 100_000); + + // Bind order mirrors the numbered placeholders in the query. + let rows = sql_query(SUMMARY_AGGREGATE_SQL) + .bind::(dimensions.enabled(SummaryDimension::Day)) + .bind::(dimensions.enabled(SummaryDimension::Device)) + .bind::(dimensions.enabled(SummaryDimension::Agent)) + .bind::(dimensions.enabled(SummaryDimension::Provider)) + .bind::(dimensions.enabled(SummaryDimension::Model)) + .bind::(dimensions.enabled(SummaryDimension::EventType)) + .bind::(user_id.to_string()) + .bind::(query.from.map_or(i64::MIN, timestamp_to_micros)) + .bind::(query.to.map_or(i64::MAX, timestamp_to_micros)) + .bind::(!agent_names.is_empty()) + .bind::(&agent_names_json) + .bind::(session_id.is_some()) + .bind::(session_id.unwrap_or_default()) + .bind::(provider.is_some()) + .bind::(provider.as_deref().unwrap_or_default()) + .bind::(model.is_some()) + .bind::(model.unwrap_or_default()) + .bind::(event_type.is_some()) + .bind::(event_type.unwrap_or_default()) + .bind::(limit) + .load::(connection)?; + let include_user = dimensions.enabled(SummaryDimension::User); + let rows = rows .into_iter() - .map(|(key, bucket)| bucket.into_row(key)) - .collect::>(); - rows.sort_by(|left, right| { - right - .total_tokens - .cmp(&left.total_tokens) - .then_with(|| left.agent_name.cmp(&right.agent_name)) - .then_with(|| left.user_email.cmp(&right.user_email)) - .then_with(|| left.day.cmp(&right.day)) - }); - let row_limit = usize::try_from(normalize_limit(query.limit, scan_limit, 100_000)) - .map_err(|error| AppError::internal(format!("invalid summary row limit: {error}")))?; - rows.truncate(row_limit); + .map(|row| row.into_summary_row(include_user.then_some((user_id, &owner)))) + .collect::, AppError>>()?; Ok(SummaryResponse { from: query.from, to: query.to, @@ -1002,6 +1022,128 @@ pub(super) fn usage_summary( }) } +/// Usage totals grouped in SQL, reading only the columns a summary reports. +/// +/// Event text and metadata hold whole request bodies, so loading rows to sum +/// them in memory read gigabytes for a week of usage. The owner is the single +/// standalone user, so the user dimension is filled in afterwards. +/// +/// Bind order: `?1`..`?6` day, device, Agent, provider, model, and event-type +/// dimensions; `?7` owner; `?8`/`?9` observation bounds in microseconds, kept +/// unconditional so the owner-time index bounds the scan; `?10`/`?11` Agent +/// filter flag and JSON name list; `?12`..`?19` session, provider, model, and +/// event-type filter flags and values; `?20` row limit. +const SUMMARY_AGGREGATE_SQL: &str = "\ + SELECT \ + CASE WHEN ?1 THEN strftime('%Y-%m-%d', e.observed_at / 1000000, 'unixepoch') END AS day, \ + CASE WHEN ?2 THEN d.host_name END AS host_name, \ + CASE WHEN ?2 THEN d.platform END AS platform, \ + CASE WHEN ?2 THEN d.os_version END AS os_version, \ + CASE \ + WHEN ?3 THEN CASE \ + WHEN e.agent_name IN ('claude', 'claude-desktop') THEN 'claude-code' \ + ELSE e.agent_name \ + END \ + END AS agent_name, \ + CASE WHEN ?4 THEN e.llm_provider END AS llm_provider, \ + CASE WHEN ?5 THEN e.llm_model END AS llm_model, \ + CASE WHEN ?6 THEN e.event_type END AS event_type, \ + COUNT(DISTINCT e.session_pk) AS sessions, \ + COUNT(DISTINCT e.turn_pk) AS turns, \ + COUNT(*) FILTER (WHERE e.event_type = 'request') AS requests, \ + COUNT(*) FILTER (WHERE e.event_type = 'response') AS responses, \ + COALESCE(SUM(e.input_tokens), 0) AS input_tokens, \ + COALESCE(SUM(e.output_tokens), 0) AS output_tokens, \ + COALESCE(SUM(e.cache_read_tokens), 0) AS cache_read_tokens, \ + COALESCE(SUM(e.cache_write_tokens), 0) AS cache_write_tokens, \ + COALESCE(SUM(e.reasoning_tokens), 0) AS reasoning_tokens, \ + COALESCE(SUM(e.total_tokens), 0) AS total_tokens \ + FROM llm_usage_events e \ + LEFT JOIN devices d ON ?2 AND d.id = e.device_context_id \ + WHERE e.user_id = ?7 \ + AND e.observed_at >= ?8 \ + AND e.observed_at < ?9 \ + AND (?10 = 0 OR e.agent_name IN (SELECT value FROM json_each(?11))) \ + AND (?12 = 0 OR e.session_id = ?13) \ + AND (?14 = 0 OR e.llm_provider = ?15) \ + AND (?16 = 0 OR e.llm_model = ?17) \ + AND (?18 = 0 OR e.event_type = ?19) \ + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8 \ + ORDER BY total_tokens DESC, agent_name ASC, day ASC \ + LIMIT ?20"; + +#[derive(QueryableByName)] +struct SummaryAggregateRow { + #[diesel(sql_type = Nullable)] + day: Option, + #[diesel(sql_type = Nullable)] + host_name: Option, + #[diesel(sql_type = Nullable)] + platform: Option, + #[diesel(sql_type = Nullable)] + os_version: Option, + #[diesel(sql_type = Nullable)] + agent_name: Option, + #[diesel(sql_type = Nullable)] + llm_provider: Option, + #[diesel(sql_type = Nullable)] + llm_model: Option, + #[diesel(sql_type = Nullable)] + event_type: Option, + #[diesel(sql_type = BigInt)] + sessions: i64, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + requests: i64, + #[diesel(sql_type = BigInt)] + responses: i64, + #[diesel(sql_type = BigInt)] + input_tokens: i64, + #[diesel(sql_type = BigInt)] + output_tokens: i64, + #[diesel(sql_type = BigInt)] + cache_read_tokens: i64, + #[diesel(sql_type = BigInt)] + cache_write_tokens: i64, + #[diesel(sql_type = BigInt)] + reasoning_tokens: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, +} + +impl SummaryAggregateRow { + fn into_summary_row(self, user: Option<(Uuid, &OwnerRecord)>) -> Result { + let count = |value: i64, name: &str| { + usize::try_from(value) + .map_err(|error| AppError::internal(format!("invalid summary {name}: {error}"))) + }; + Ok(SummaryRow { + day: self.day, + user_id: user.map(|(user_id, _)| user_id), + user_name: user.and_then(|(_, owner)| owner.name.clone()), + user_email: user.map(|(_, owner)| owner.email.clone()), + host_name: self.host_name, + platform: self.platform, + os_version: self.os_version, + agent_name: self.agent_name, + llm_provider: self.llm_provider, + llm_model: self.llm_model, + event_type: self.event_type, + sessions: count(self.sessions, "sessions")?, + turns: count(self.turns, "turns")?, + requests: self.requests, + responses: self.responses, + input_tokens: self.input_tokens, + output_tokens: self.output_tokens, + cache_read_tokens: self.cache_read_tokens, + cache_write_tokens: self.cache_write_tokens, + reasoning_tokens: self.reasoning_tokens, + total_tokens: self.total_tokens, + }) + } +} + struct OwnerRecord { email: String, name: Option, @@ -1088,126 +1230,6 @@ impl SummaryDimensions { } } -#[derive(Hash, Eq, PartialEq)] -struct SummaryKey { - day: Option, - user_id: Option, - user_name: Option, - user_email: Option, - host_name: Option, - platform: Option, - os_version: Option, - agent_name: Option, - llm_provider: Option, - llm_model: Option, - event_type: Option, -} - -impl SummaryKey { - fn new( - event: &EventRecord, - device: Option<&DeviceRecord>, - user_id: Uuid, - owner: &OwnerRecord, - dimensions: &SummaryDimensions, - ) -> Self { - let include_user = dimensions.enabled(SummaryDimension::User); - let include_device = dimensions.enabled(SummaryDimension::Device); - Self { - day: dimensions - .enabled(SummaryDimension::Day) - .then(|| event.observed_at.format("%Y-%m-%d").to_string()), - user_id: include_user.then_some(user_id), - user_name: include_user.then(|| owner.name.clone()).flatten(), - user_email: include_user.then(|| owner.email.clone()), - host_name: include_device - .then(|| device.map(|value| value.host_name.clone())) - .flatten(), - platform: include_device - .then(|| device.map(|value| value.platform.clone())) - .flatten(), - os_version: include_device - .then(|| device.and_then(|value| value.os_version.clone())) - .flatten(), - agent_name: dimensions - .enabled(SummaryDimension::Agent) - .then(|| canonical_agent_name(&event.agent_name)), - llm_provider: dimensions - .enabled(SummaryDimension::Provider) - .then(|| event.llm_provider.clone()), - llm_model: dimensions - .enabled(SummaryDimension::Model) - .then(|| event.llm_model.clone()), - event_type: dimensions - .enabled(SummaryDimension::EventType) - .then(|| event.event_type.clone()), - } - } -} - -#[derive(Default)] -struct SummaryBucket { - sessions: HashSet, - turns: HashSet, - requests: i64, - responses: i64, - input_tokens: i64, - output_tokens: i64, - cache_read_tokens: i64, - cache_write_tokens: i64, - reasoning_tokens: i64, - total_tokens: i64, -} - -impl SummaryBucket { - fn add(&mut self, event: &EventRecord) { - self.sessions.insert(event.session_pk); - self.turns.insert(event.turn_pk); - self.requests = self - .requests - .saturating_add(i64::from(event.event_type == "request")); - self.responses = self - .responses - .saturating_add(i64::from(event.event_type == "response")); - self.input_tokens = self.input_tokens.saturating_add(event.input_tokens); - self.output_tokens = self.output_tokens.saturating_add(event.output_tokens); - self.cache_read_tokens = self - .cache_read_tokens - .saturating_add(event.cache_read_tokens); - self.cache_write_tokens = self - .cache_write_tokens - .saturating_add(event.cache_write_tokens); - self.reasoning_tokens = self.reasoning_tokens.saturating_add(event.reasoning_tokens); - self.total_tokens = self.total_tokens.saturating_add(event.total_tokens); - } - - fn into_row(self, key: SummaryKey) -> SummaryRow { - SummaryRow { - day: key.day, - user_id: key.user_id, - user_name: key.user_name, - user_email: key.user_email, - host_name: key.host_name, - platform: key.platform, - os_version: key.os_version, - agent_name: key.agent_name, - llm_provider: key.llm_provider, - llm_model: key.llm_model, - event_type: key.event_type, - sessions: self.sessions.len(), - turns: self.turns.len(), - requests: self.requests, - responses: self.responses, - input_tokens: self.input_tokens, - output_tokens: self.output_tokens, - cache_read_tokens: self.cache_read_tokens, - cache_write_tokens: self.cache_write_tokens, - reasoning_tokens: self.reasoning_tokens, - total_tokens: self.total_tokens, - } - } -} - fn load_device_contexts( connection: &mut SqliteConnection, events: &[EventRecord], @@ -1330,11 +1352,14 @@ mod tests { use serde_json::json; use uuid::Uuid; - use crate::{search::SessionSearchQuery, usage::IngestEventsRequest}; + use crate::{ + search::SessionSearchQuery, + usage::{IngestEventsRequest, SummaryQuery}, + }; use super::{ super::{migrations, schema::llm_usage_events, search}, - ingest_events, session_outline, + ingest_events, session_outline, usage_summary, }; const OWNER_ID: Uuid = Uuid::from_u128(1); @@ -1484,6 +1509,137 @@ mod tests { assert_eq!(after.turns[0].events[0].metadata, expected); } + fn summary_seeded() -> SqliteConnection { + let mut database = database(); + let event = |id: &str, agent: &str, session: &str, turn: i32, observed_at: &str| { + let (provider, model) = if agent == "Codex" { + ("OpenAI", "gpt-5") + } else { + ("Anthropic", "claude-x") + }; + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos", "os_version": "15.1"}, + "agent": {"name": agent}, + "session_id": session, + "turn_index": turn, + "llm": {"provider": provider, "model": model}, + "event_type": if id.ends_with("req") { "request" } else { "response" }, + "text": "body", + "token_usage": if id.ends_with("req") { + json!({"input_tokens": 10_i64, "cache_read_tokens": 4_i64}) + } else { + json!({"output_tokens": 5_i64}) + }, + "metadata": {"raw_body": "large"} + }) + }; + let request: IngestEventsRequest = serde_json::from_value(json!({ + "events": [ + event("a-req", "Codex", "alpha", 1_i32, "2026-09-01T23:00:00Z"), + event("a-res", "Codex", "alpha", 1_i32, "2026-09-01T23:30:00Z"), + event("b-req", "claude-desktop", "beta", 1_i32, "2026-09-02T01:00:00Z"), + event("c-res", "claude-code", "gamma", 1_i32, "2026-09-02T02:00:00Z"), + event("a2-req", "Codex", "alpha", 2_i32, "2026-09-03T00:00:00Z"), + ] + })) + .expect("request should deserialize"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn summary(database: &mut SqliteConnection, query: serde_json::Value) -> serde_json::Value { + let query: SummaryQuery = serde_json::from_value(query).expect("query should deserialize"); + let response = + usage_summary(database, &query, OWNER_ID, 500).expect("summary should succeed"); + serde_json::to_value(response.rows).expect("rows should serialize") + } + + #[test] + fn summary_groups_in_sql_with_canonical_agents_and_distinct_counts() { + let mut database = summary_seeded(); + + let rows = summary( + &mut database, + json!({"group_by": "agent,provider,model", "to": "2026-09-03T00:00:00Z"}), + ); + + assert_eq!( + rows, + json!([ + // Equal totals fall back to the Agent name. + { + "agent_name": "claude-code", "llm_provider": "anthropic", + "llm_model": "claude-x", + "sessions": 2_i64, "turns": 2_i64, "requests": 1_i64, "responses": 1_i64, + "input_tokens": 10_i64, "output_tokens": 5_i64, "cache_read_tokens": 4_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 19_i64 + }, + { + "agent_name": "codex", "llm_provider": "openai", "llm_model": "gpt-5", + "sessions": 1_i64, "turns": 1_i64, "requests": 1_i64, "responses": 1_i64, + "input_tokens": 10_i64, "output_tokens": 5_i64, "cache_read_tokens": 4_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 19_i64 + } + ]) + ); + } + + #[test] + fn summary_filters_and_groups_by_utc_day_and_event_type() { + let mut database = summary_seeded(); + + let rows = summary( + &mut database, + json!({"group_by": "day,event_type", "agent_name": "claude", "limit": 10_i64}), + ); + + let mut buckets = rows + .as_array() + .expect("rows should be an array") + .iter() + .map(|row| { + ( + row["day"].clone(), + row["event_type"].clone(), + row["total_tokens"].clone(), + ) + }) + .collect::>(); + buckets.sort_by_key(|bucket| bucket.1.to_string()); + assert_eq!( + buckets, + vec![ + (json!("2026-09-02"), json!("request"), json!(14_i64)), + (json!("2026-09-02"), json!("response"), json!(5_i64)), + ] + ); + let codex_requests = summary( + &mut database, + json!({"group_by": "event_type", "agent_name": "codex", "event_type": "request"}), + ); + assert_eq!(codex_requests[0]["requests"], json!(2_i64)); + assert_eq!(codex_requests[0]["turns"], json!(2_i64)); + assert_eq!(codex_requests.as_array().map(Vec::len), Some(1)); + } + + #[test] + fn summary_fills_owner_and_device_dimensions() { + let mut database = summary_seeded(); + + let rows = summary(&mut database, json!({"group_by": "user,device"})); + + assert_eq!(rows.as_array().map(Vec::len), Some(1)); + assert_eq!(rows[0]["user_id"], json!(OWNER_ID)); + assert!(rows[0]["user_email"].is_string()); + assert_eq!(rows[0]["host_name"], json!("alice-mbp")); + assert_eq!(rows[0]["platform"], json!("macos")); + assert_eq!(rows[0]["os_version"], json!("15.1")); + assert_eq!(rows[0]["sessions"], json!(3_i64)); + assert!(rows[0].get("agent_name").is_none()); + } + fn request_with_text(text: &str) -> IngestEventsRequest { serde_json::from_value(json!({ "events": [{ From ba0b55f00cfa6683b61e588671abb21e30708682 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 02:11:52 -0700 Subject: [PATCH 18/30] perf(sqlite): answer summaries and the session list from covering indexes Token columns sit after the event text in each row, so reading them walks the text's overflow pages. Two indexes carry every column the usage summary and the session list read, and the session list counts an indexed column, so both queries no longer touch event rows. Co-Authored-By: Claude Opus 5.5 --- .../20261002000000_usage_covering_indexes/down.sql | 2 ++ .../20261002000000_usage_covering_indexes/up.sql | 13 +++++++++++++ .../src/storage/sqlite/session_list.rs | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql create mode 100644 crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql diff --git a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql new file mode 100644 index 0000000..df4680b --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql @@ -0,0 +1,2 @@ +DROP INDEX llm_usage_events_session_activity_idx; +DROP INDEX llm_usage_events_summary_cover_idx; diff --git a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql new file mode 100644 index 0000000..f666be5 --- /dev/null +++ b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql @@ -0,0 +1,13 @@ +-- Event rows hold whole request bodies in text and metadata, so reading a +-- token column from the row walks through megabytes of overflow pages. These +-- indexes carry every column the usage summary and the session list read, +-- letting both answer from the index alone. +CREATE INDEX llm_usage_events_summary_cover_idx + ON llm_usage_events ( + user_id, observed_at, agent_name, llm_provider, llm_model, event_type, + device_context_id, session_pk, turn_pk, session_id, + input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, + reasoning_tokens, total_tokens + ); +CREATE INDEX llm_usage_events_session_activity_idx + ON llm_usage_events (session_pk, observed_at, total_tokens); diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index 095f5db..3705559 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -34,7 +34,7 @@ const ACTIVITY_CTE: &str = "\ s.ended_at AS ended_at, \ COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id) AS turns, \ - COUNT(e.id) AS events, \ + COUNT(e.session_pk) AS events, \ COALESCE(SUM(e.total_tokens), 0) AS total_tokens, \ st.title AS title, \ st.purpose AS purpose \ From 1b5626de380911d539f77a5b3e82f03739f86d03 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 02:35:32 -0700 Subject: [PATCH 19/30] feat(trajectory): expose usage for every request and response --- .../src/storage/postgres/repository.rs | 4 +- crates/abyss-backend/src/usage/trajectory.rs | 133 ++++++++++++++++++ scripts/tests/blackbox_api.py | 26 ++++ 3 files changed, 161 insertions(+), 2 deletions(-) diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index a378a9b..152756c 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -1755,8 +1755,8 @@ mod tests { let tokens = normalized_tokens(&event).expect("token usage should normalize"); assert_eq!( - tokens.total, 34_i64, - "missing total_tokens should be derived from all token components" + tokens.total, 29_i64, + "missing total_tokens should sum disjoint counters without adding reasoning twice" ); } diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 442f879..dad6da1 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -38,6 +38,74 @@ pub struct SessionTrajectoryResponse { pub session: SessionInfo, /// Items in authoritative event order. pub items: Vec, + /// Every observed request and response, including events with no new content. + pub events: Vec, +} + +#[derive(Debug, Serialize)] +/// Lightweight usage summary independent of content de-duplication. +pub struct TrajectoryEvent { + /// Event that owns these counters. + pub event_pk: Uuid, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Canonical request or response value. + pub event_type: String, + /// Observation time of the event. + pub observed_at: DateTime, + /// Model requested or returned by the provider. + pub llm_model: String, + /// Uncached input tokens; cache counters are separate. + pub input_tokens: i64, + /// Output tokens, including reasoning. + pub output_tokens: i64, + /// Input tokens read from a provider cache. + pub cache_read_tokens: i64, + /// Input tokens written into a provider cache. + pub cache_write_tokens: i64, + /// Reasoning breakdown of output tokens. + pub reasoning_tokens: i64, + /// Authoritative total for this event, counted once. + pub total_tokens: i64, + /// Whether counters were reported or estimated rather than absent or hidden. + pub token_usage_available: bool, + /// Whether the collector estimated token usage. + pub token_usage_estimated: bool, +} + +impl From<&UsageEventResponse> for TrajectoryEvent { + fn from(event: &UsageEventResponse) -> Self { + let source = event + .metadata + .get("token_usage_source") + .and_then(Value::as_str); + let hidden = + event.metadata.pointer("/content_policy/token_usage") == Some(&Value::Bool(false)); + let has_usage = event.input_tokens != 0 + || event.output_tokens != 0 + || event.cache_read_tokens != 0 + || event.cache_write_tokens != 0 + || event.reasoning_tokens != 0 + || event.total_tokens != 0; + Self { + event_pk: event.id, + turn_index: event.turn_index, + event_type: event.event_type.clone(), + observed_at: event.observed_at, + llm_model: event.llm_model.clone(), + input_tokens: event.input_tokens, + output_tokens: event.output_tokens, + cache_read_tokens: event.cache_read_tokens, + cache_write_tokens: event.cache_write_tokens, + reasoning_tokens: event.reasoning_tokens, + total_tokens: event.total_tokens, + token_usage_available: !hidden + && source != Some("absent") + && (has_usage || source.is_some()), + token_usage_estimated: event.metadata.get("token_usage_estimated") + == Some(&Value::Bool(true)), + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -142,8 +210,10 @@ impl TrajectoryItem { /// Merges an ordered session timeline into its trajectory. pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { let mut merger = Merger::default(); + let mut events = Vec::new(); for turn in &timeline.turns { for event in &turn.events { + events.push(TrajectoryEvent::from(event)); if event.event_type == "request" { merger.request(event); } else { @@ -154,6 +224,7 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { SessionTrajectoryResponse { session: timeline.session, items: merger.items, + events, } } @@ -358,6 +429,68 @@ fn segment_str<'a>(segment: &'a Value, key: &str) -> Option<&'a str> { #[cfg(test)] mod tests { + #[test] + fn usage_retains_events_without_new_content_and_counts_each_event_once() { + let body = r#"{"input":[{"role":"user","content":"hello"}]}"#; + let mut request = raw_event("request", body); + request.input_tokens = 2; + request.cache_read_tokens = 77_000; + request.cache_write_tokens = 400; + request.total_tokens = 77_402; + let repeated = raw_event("request", body); + let mut response = event("response", "", json!({"token_usage_estimated": true})); + response.llm_model = "other-model".to_owned(); + response.output_tokens = 150; + response.reasoning_tokens = 40; + response.total_tokens = 150; + let expected_ids = vec![request.id, repeated.id, response.id]; + + let trajectory = build(timeline(vec![request, repeated, response])); + assert_eq!(trajectory.items.len(), 1); + assert_eq!( + trajectory + .events + .iter() + .map(|event| event.event_pk) + .collect::>(), + expected_ids + ); + assert_eq!(trajectory.events[0].total_tokens, 77_402); + assert_eq!(trajectory.events[0].cache_read_tokens, 77_000); + assert_eq!(trajectory.events[0].cache_write_tokens, 400); + assert!(trajectory.events[0].token_usage_available); + assert!(!trajectory.events[1].token_usage_available); + assert_eq!(trajectory.events[2].llm_model, "other-model"); + assert_eq!(trajectory.events[2].output_tokens, 150); + assert_eq!(trajectory.events[2].reasoning_tokens, 40); + assert!(trajectory.events[2].token_usage_estimated); + let serialized = serde_json::to_value(&trajectory.events).unwrap(); + assert!(serialized[0].get("text").is_none()); + assert!(serialized[0].get("metadata").is_none()); + } + + #[test] + fn usage_distinguishes_reported_zero_from_absent_and_hidden_counts() { + let reported = event( + "response", + "", + json!({"token_usage_source": "provider_reported"}), + ); + let absent = event("request", "", json!({"token_usage_source": "absent"})); + let mut hidden = event( + "request", + "", + json!({"content_policy": {"token_usage": false}}), + ); + hidden.input_tokens = 10; + hidden.total_tokens = 10; + let trajectory = build(timeline(vec![reported, absent, hidden])); + assert_eq!(trajectory.events.len(), 3); + assert!(trajectory.events[0].token_usage_available); + assert!(!trajectory.events[1].token_usage_available); + assert!(!trajectory.events[2].token_usage_available); + } + use serde_json::json; use super::*; diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index a55bf85..f1eec24 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -266,6 +266,32 @@ def run(base_url: str, token: str, run_id: str) -> None: for turn in timeline.json()["turns"] for item in turn["events"] ] + trajectory = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{session_pk}/trajectory", + token=token, + ) + require_status(trajectory, 200, "session trajectory") + usage_events = trajectory.json()["events"] + require( + [item["event_pk"] for item in usage_events] + == [item["id"] for item in timeline_events], + "trajectory usage must preserve every event in authoritative order", + ) + for usage_event, timeline_event in zip(usage_events, timeline_events): + for field in ( + "turn_index", "event_type", "observed_at", "llm_model", + "input_tokens", "output_tokens", "cache_read_tokens", + "cache_write_tokens", "reasoning_tokens", "total_tokens", + ): + require( + usage_event[field] == timeline_event[field], + f"trajectory usage {field} must match the authoritative event", + ) + require(usage_event["token_usage_available"], "nonzero usage must be available") + require("text" not in usage_event and "metadata" not in usage_event, + "trajectory usage summaries must omit content bodies") require( [item["event_id"] for item in timeline_events] == [request_event_id, response_event_id], From c6bd7b56f53a42043d3bae02601558ebf15cd32b Mon Sep 17 00:00:00 2001 From: Auceptin Date: Fri, 2 Oct 2026 04:22:28 -0700 Subject: [PATCH 20/30] feat(events): add a content-free list projection Event text and metadata hold whole request bodies, so ten events came to about 8 MB while an event list shows a few hundred bytes of each. With fields=list the events endpoint returns what a list shows, and neither storage backend reads text, its digest, or metadata, nor loads attachments. Without the parameter the response is unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/api.rs | 10 +- .../src/storage/postgres/repository.rs | 46 +++++- .../src/storage/sqlite/repository.rs | 86 +++++++++-- crates/abyss-backend/src/usage/mod.rs | 143 +++++++++++++++++- scripts/tests/blackbox_api.py | 25 +++ 5 files changed, 289 insertions(+), 21 deletions(-) diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index 3f24eb0..8e1b355 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -25,7 +25,7 @@ use crate::{ search::{SessionSearchQuery, SessionSearchResponse}, storage::StorageBackend, usage::{ - IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, + IngestEventsRequest, IngestEventsResponse, RawEventListResponse, RawEventsQuery, SessionOutlineResponse, SessionTimelineResponse, SummaryFields, SummaryQuery, TokenUsageSummaryResponse, UsageEventResponse, session_list::{SessionListQuery, SessionListResponse}, @@ -261,14 +261,18 @@ async fn raw_events( State(state): State, headers: HeaderMap, Query(query): Query, -) -> Result, AppError> { +) -> Result { let user_id = state.identity.authenticate(&headers)?; let default_page_size = state.default_page_size; + let includes_content = query.includes_content(); let response = state .storage .raw_events(user_id, query, default_page_size) .await?; - Ok(Json(response)) + if !includes_content { + return Ok(Json(RawEventListResponse::from_events(response)).into_response()); + } + Ok(Json(response).into_response()) } #[derive(Serialize)] diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index 34494c8..391d23a 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -15,7 +15,7 @@ use diesel::{ result::OptionalExtension, sql_query, sql_types::{ - Array, BigInt, Bool, Integer, Jsonb, Nullable, Text, Timestamptz, Uuid as SqlUuid, + Array, BigInt, Bool, Bytea, Integer, Jsonb, Nullable, Text, Timestamptz, Uuid as SqlUuid, }, }; use uuid::Uuid; @@ -578,6 +578,7 @@ pub fn raw_events( limit: page_size.saturating_add(1_i64), offset, }, + query.includes_content(), )?; // Fetch one sentinel row beyond the requested page so no count query is // needed merely to determine whether a next offset exists. @@ -586,7 +587,11 @@ pub fn raw_events( events.truncate(page_size_usize); } let device_contexts = load_device_contexts_for_events(connection, &events)?; - let mut attachments_by_event = load_attachments_for_events(connection, &events)?; + let mut attachments_by_event = if query.includes_content() { + load_attachments_for_events(connection, &events)? + } else { + HashMap::new() + }; Ok(RawEventsResponse { events: events @@ -1019,9 +1024,12 @@ fn upsert_turn( .map_err(AppError::from) } +/// Loads matching events newest first. Without content, text and metadata are +/// not read: they hold whole request bodies, and an event list never shows them. fn load_filtered_events( connection: &mut PgConnection, filters: &EventFilters<'_>, + include_content: bool, ) -> Result, AppError> { let event_type = filters .event_type @@ -1065,10 +1073,40 @@ fn load_filtered_events( query = query.filter(llm_usage_events::event_type.eq(event_type)); } - query + let query = query .order(llm_usage_events::observed_at.desc()) .limit(filters.limit) - .offset(filters.offset) + .offset(filters.offset); + if include_content { + return query.load::(connection).map_err(AppError::from); + } + query + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::device_context_id, + llm_usage_events::session_pk, + llm_usage_events::turn_pk, + llm_usage_events::event_id, + llm_usage_events::agent_name, + llm_usage_events::agent_version, + llm_usage_events::session_id, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + sql::>("NULL"), + sql::>("NULL"), + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + sql::("'{}'::jsonb"), + llm_usage_events::created_at, + )) .load::(connection) .map_err(AppError::from) } diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index 6628e20..e9b9694 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -8,7 +8,7 @@ use diesel::{ dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, - sql_types::{BigInt, Bool, Nullable, Text}, + sql_types::{BigInt, Binary, Bool, Nullable, Text}, }; use uuid::Uuid; @@ -552,13 +552,18 @@ pub(super) fn raw_events( limit: page_size.saturating_add(1), offset, }, + query.includes_content(), )?; let has_next_page = events.len() > page_size_usize; if has_next_page { events.truncate(page_size_usize); } let devices = load_device_contexts(connection, &events)?; - let mut attachments = load_attachments(connection, &events)?; + let mut attachments = if query.includes_content() { + load_attachments(connection, &events)? + } else { + HashMap::new() + }; let responses = events .into_iter() .map(|event| { @@ -593,9 +598,12 @@ struct EventFilters<'a> { offset: i64, } +/// Loads matching events newest first. Without content, text and metadata are +/// not read: they hold whole request bodies, and an event list never shows them. fn load_filtered_events( connection: &mut SqliteConnection, filters: &EventFilters<'_>, + include_content: bool, ) -> Result, AppError> { let mut query = llm_usage_events::table .filter(llm_usage_events::user_id.eq(filters.user_id.to_string())) @@ -631,18 +639,48 @@ fn load_filtered_events( if let Some(event_type) = filters.event_type.and_then(non_empty) { query = query.filter(llm_usage_events::event_type.eq(validate_event_type(event_type)?)); } - query + let query = query .order_by(( llm_usage_events::observed_at.desc(), llm_usage_events::id.desc(), )) .limit(filters.limit) - .offset(filters.offset) - .select(EventRow::as_select()) - .load::(connection)? - .into_iter() - .map(EventRow::into_record) - .collect() + .offset(filters.offset); + let rows = if include_content { + query + .select(EventRow::as_select()) + .load::(connection)? + } else { + query + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::device_context_id, + llm_usage_events::session_pk, + llm_usage_events::turn_pk, + llm_usage_events::event_id, + llm_usage_events::agent_name, + llm_usage_events::agent_version, + llm_usage_events::session_id, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + sql::>("NULL"), + sql::>("NULL"), + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + sql::("'{}'"), + llm_usage_events::created_at, + )) + .load::(connection)? + }; + rows.into_iter().map(EventRow::into_record).collect() } pub(super) fn session_timeline( @@ -1346,12 +1384,12 @@ mod tests { use crate::{ search::SessionSearchQuery, - usage::{IngestEventsRequest, SummaryQuery}, + usage::{IngestEventsRequest, RawEventsQuery, SummaryQuery}, }; use super::{ super::{migrations, schema::llm_usage_events, search}, - ingest_events, session_outline, usage_summary, + ingest_events, raw_events, session_outline, usage_summary, }; const OWNER_ID: Uuid = Uuid::from_u128(1); @@ -1501,6 +1539,32 @@ mod tests { assert_eq!(after.turns[0].events[0].metadata, expected); } + #[test] + fn event_list_leaves_out_content_but_keeps_what_a_list_shows() { + let mut database = summary_seeded(); + let page = |database: &mut SqliteConnection, query: serde_json::Value| { + let query: RawEventsQuery = + serde_json::from_value(query).expect("query should deserialize"); + raw_events(database, &query, OWNER_ID, 50).expect("events should load") + }; + + let list = page(&mut database, json!({"fields": "list", "limit": 2_i64})); + let full = page(&mut database, json!({"limit": 2_i64})); + + assert_eq!(list.next_page_token.as_deref(), Some("2")); + assert_eq!(list.events.len(), 2); + for (listed, complete) in list.events.iter().zip(&full.events) { + assert_eq!(listed.id, complete.id); + assert_eq!(listed.total_tokens, complete.total_tokens); + assert_eq!(listed.host_name.as_deref(), Some("alice-mbp")); + assert_eq!(listed.text, None); + assert_eq!(listed.metadata, json!({})); + assert!(listed.attachments.is_empty()); + assert_eq!(complete.text.as_deref(), Some("body")); + assert_eq!(complete.metadata["raw_body"], json!("large")); + } + } + fn summary_seeded() -> SqliteConnection { let mut database = database(); let event = |id: &str, agent: &str, session: &str, turn: i32, observed_at: &str| { diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 8e94dc8..54d813c 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -365,6 +365,27 @@ pub struct RawEventsQuery { pub limit: Option, /// Number of newest matching events to skip. pub offset: Option, + /// Selects complete events or the content-free list projection. + pub fields: Option, +} + +impl RawEventsQuery { + /// Whether events carry their text, metadata, and attachments. + pub fn includes_content(&self) -> bool { + self.fields.unwrap_or_default() == RawEventFields::Full + } +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +/// Response projection requested from the raw-event endpoint. +pub enum RawEventFields { + /// Return every event field, including text, metadata, and attachments. + #[default] + Full, + /// Return what an event list shows. Text and metadata hold whole request + /// bodies, so storage does not read them and attachments are not loaded. + List, } #[derive(Debug, Serialize)] @@ -376,6 +397,110 @@ pub struct RawEventsResponse { pub next_page_token: Option, } +#[derive(Debug, Serialize)] +/// One page of usage events without their content, for event lists. +pub struct RawEventListResponse { + /// Events ordered from newest to oldest observation time. + pub events: Vec, + /// Decimal offset for the next page, or `None` at the end. + pub next_page_token: Option, +} + +impl RawEventListResponse { + /// Drops the content fields from a page loaded without them. + pub fn from_events(response: RawEventsResponse) -> Self { + Self { + events: response + .events + .into_iter() + .map(UsageEventListItem::from) + .collect(), + next_page_token: response.next_page_token, + } + } +} + +#[derive(Debug, Serialize)] +/// One usage event as an event list shows it: identity, model, Agent, device, +/// and token counts, without text, metadata, or attachments. +pub struct UsageEventListItem { + /// Backend-generated event primary key. + pub id: Uuid, + /// Collector-generated idempotency key. + pub event_id: String, + /// Authenticated owner identifier. + pub user_id: Uuid, + /// Backend device-context primary key. + pub device_context_id: Uuid, + /// Device host name, when the referenced device still exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub host_name: Option, + /// Device platform, when the referenced device still exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, + /// Backend session primary key. + pub session_pk: Uuid, + /// Backend turn primary key. + pub turn_pk: Uuid, + /// Canonical Agent name. + pub agent_name: String, + /// Agent version observed on this event. + pub agent_version: Option, + /// Agent-native session identifier. + pub session_id: String, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Canonical LLM provider slug. + pub llm_provider: String, + /// Provider-specific model identifier. + pub llm_model: String, + /// Canonical request or response value. + pub event_type: String, + /// Normalized input-token count. + pub input_tokens: i64, + /// Normalized output-token count. + pub output_tokens: i64, + /// Normalized cache-read-token count. + pub cache_read_tokens: i64, + /// Normalized cache-write-token count. + pub cache_write_tokens: i64, + /// Normalized reasoning-token count. + pub reasoning_tokens: i64, + /// Provider-reported or derived total-token count. + pub total_tokens: i64, + /// Collector observation timestamp. + pub observed_at: DateTime, +} + +impl From for UsageEventListItem { + fn from(event: UsageEventResponse) -> Self { + Self { + id: event.id, + event_id: event.event_id, + user_id: event.user_id, + device_context_id: event.device_context_id, + host_name: event.host_name, + platform: event.platform, + session_pk: event.session_pk, + turn_pk: event.turn_pk, + agent_name: event.agent_name, + agent_version: event.agent_version, + session_id: event.session_id, + turn_index: event.turn_index, + llm_provider: event.llm_provider, + llm_model: event.llm_model, + event_type: event.event_type, + input_tokens: event.input_tokens, + output_tokens: event.output_tokens, + cache_read_tokens: event.cache_read_tokens, + cache_write_tokens: event.cache_write_tokens, + reasoning_tokens: event.reasoning_tokens, + total_tokens: event.total_tokens, + observed_at: event.observed_at, + } + } +} + #[derive(Debug, Serialize)] /// API representation of one persisted usage event. pub struct UsageEventResponse { @@ -590,9 +715,9 @@ pub fn empty_metadata() -> Value { #[cfg(test)] mod tests { use super::{ - IngestEventsRequest, SummaryFields, SummaryQuery, SummaryResponse, SummaryRow, - TokenUsageSummaryResponse, UsageEventType, session_metadata_from_event, - session_working_directory, + IngestEventsRequest, RawEventFields, RawEventsQuery, SummaryFields, SummaryQuery, + SummaryResponse, SummaryRow, TokenUsageSummaryResponse, UsageEventType, + session_metadata_from_event, session_working_directory, }; use serde_json::json; @@ -731,6 +856,18 @@ mod tests { assert_eq!(query.fields, Some(SummaryFields::TokenUsage)); } + #[test] + fn raw_events_query_selects_the_list_projection() { + let list = serde_json::from_value::(json!({"fields": "list"})) + .expect("list query should deserialize"); + let full = serde_json::from_value::(json!({})) + .expect("default query should deserialize"); + + assert_eq!(list.fields, Some(RawEventFields::List)); + assert!(!list.includes_content()); + assert!(full.includes_content()); + } + #[test] fn token_usage_summary_response_omits_non_token_fields() { let response = TokenUsageSummaryResponse::from_summary(SummaryResponse { diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index 01efabb..9d9b533 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -216,6 +216,31 @@ def run(base_url: str, token: str, run_id: str) -> None: "raw event responses must not expose attachment content", ) + # The list projection carries what an event list shows and no content. + list_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + {"session_id": session_id, "limit": 10, "fields": "list"} + ) + list_response = request(base_url, "GET", list_path, token=token) + require_status(list_response, 200, "event list query") + listed_events = list_response.json()["events"] + require( + [item["id"] for item in listed_events] == [item["id"] for item in raw_events], + "event list must return the same events in the same order", + ) + content_fields = {"text", "text_sha256", "metadata", "attachments"} + require( + all(content_fields.isdisjoint(item) for item in listed_events), + "event list items must leave out text, metadata, and attachments", + ) + require( + all( + item["total_tokens"] == by_event_id[item["event_id"]]["total_tokens"] + and item["host_name"] == by_event_id[item["event_id"]]["host_name"] + for item in listed_events + ), + "event list items must keep token counts and device labels", + ) + attachments = by_event_id[request_event_id]["attachments"] require(len(attachments) == 1, "request event must expose one attachment record") require(attachments[0]["content_available"], "attachment bytes must be persisted") From 36a8bb9e4e024099dd906f83dbc3920bbf527fd7 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Sun, 4 Oct 2026 01:24:06 -0700 Subject: [PATCH 21/30] fix(trajectory): name streams by their opening prompt alone A stream was named by every system message in a request. Claude Code puts a billing header with a per-request hash and previous-request id at the top of its system prompt, a context budget at its end, and system messages inside the conversation that grow with it, so nearly every request opened a new stream and replayed the whole conversation. A 480-turn session came to 1,255 streams and 349,000 items, 227 MB, with one user message 2,167 times. Streams are now named by the system prompt the conversation opens with, without the billing header and context budget lines, and those lines are also left out when telling system prompts apart. The same session has 6 streams and 6,837 items, 10 MB, and keeps every distinct item it had. The archived bodies keep both lines, so billing can still be counted from them. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/usage/trajectory.rs | 69 +++++++++++++++++++ .../src/usage/trajectory/raw_body.rs | 35 +++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index dad6da1..af75eb4 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -728,6 +728,75 @@ mod tests { ); } + #[test] + fn changing_billing_headers_and_token_budgets_keep_one_stream() { + let request = |hash: &str, left: u32, messages: &str| { + format!( + r#"{{"system":[{{"type":"text","text":"x-anthropic-billing-header: cc_version=2.1.288; cch={hash}; cc_prev_req=req_{hash};"}},{{"type":"text","text":"instructions\n\n{left} tokens left"}}],"messages":{messages}}}"# + ) + }; + let items = build(timeline(vec![ + raw_event( + "request", + &request("a1", 900, r#"[{"role":"user","content":"hello"}]"#), + ), + raw_event("response", r#"{"content":[{"type":"text","text":"hi"}]}"#), + raw_event( + "request", + &request( + "b2", + 850, + r#"[{"role":"user","content":"hello"},{"role":"assistant","content":"hi"},{"role":"user","content":"next"}]"#, + ), + ), + ])) + .items; + + assert_eq!( + items + .iter() + .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + ( + TrajectoryItemKind::Context, + "x-anthropic-billing-header: cc_version=2.1.288; cch=a1; cc_prev_req=req_a1;\ninstructions\n\n900 tokens left" + ), + (TrajectoryItemKind::User, "hello"), + (TrajectoryItemKind::Assistant, "hi"), + (TrajectoryItemKind::User, "next"), + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + + #[test] + fn system_messages_inside_the_conversation_keep_one_stream() { + let first = r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"system","content":"reminder one"}]}"#; + let second = r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"system","content":"reminder one"},{"role":"assistant","content":"hi"},{"role":"system","content":"reminder two"},{"role":"user","content":"next"}]}"#; + let items = build(timeline(vec![ + raw_event("request", first), + raw_event("request", second), + ])) + .items; + + assert_eq!( + items + .iter() + .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + (TrajectoryItemKind::Context, "instructions"), + (TrajectoryItemKind::User, "hello"), + (TrajectoryItemKind::Context, "reminder one"), + (TrajectoryItemKind::Assistant, "hi"), + (TrajectoryItemKind::Context, "reminder two"), + (TrajectoryItemKind::User, "next"), + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + #[test] fn raw_compaction_keeps_existing_summary_presentation() { let request = json!({"input": [{"role": "user", "content": "write report"}, {"role": "user", "content": PROMPT}]}).to_string(); diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs index 57c779d..6548f15 100644 --- a/crates/abyss-backend/src/usage/trajectory/raw_body.rs +++ b/crates/abyss-backend/src/usage/trajectory/raw_body.rs @@ -59,10 +59,13 @@ impl RawProjection { || payload.get("instructions").is_some() || payload.get("system").is_some() { + // A stream is named by the prompt the conversation opens with. + // System messages later in the conversation are part of it: + // counting them made each new one start another stream. let context = messages .iter() - .filter(|message| message.kind() == TrajectoryItemKind::Context) - .map(|message| format!("{}: {}", message.role, message.text)) + .take_while(|message| message.kind() == TrajectoryItemKind::Context) + .map(|message| format!("{}: {}", message.role, stable_context(&message.text))) .collect::>() .join("\n\n"); let next = stream_ids.len(); @@ -96,7 +99,7 @@ impl RawProjection { let known = if context { !state .contexts - .insert((message.role.clone(), message.text.clone())) + .insert((message.role.clone(), stable_context(&message.text))) } else if request && !incremental { let found = state.history[cursor..] .iter() @@ -132,6 +135,32 @@ impl RawProjection { } } +/// Claude Code adds two lines to its system prompt that change on every +/// request: a leading billing header whose request hash and previous-request +/// id differ each time, for example `x-anthropic-billing-header: +/// cc_version=2.1.288.9e5; cc_entrypoint=cli; cch=e4f32; cc_is_subagent=true; +/// cc_prev_req=req_…; cc_prompt_id=…;`, and a trailing context budget such as +/// `14964936 tokens left`. Counting them made +/// every request a new stream that replayed the whole conversation, so +/// streams and system prompts are told apart without them. The archived body +/// keeps both, so they can still be counted for billing. +const BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:"; +const TOKEN_BUDGET_OPEN: &str = ""; +const TOKEN_BUDGET_CLOSE: &str = ""; + +/// A system prompt without the lines that change on every request. +fn stable_context(text: &str) -> String { + text.lines() + .filter(|line| !changes_every_request(line.trim())) + .collect::>() + .join("\n") +} + +fn changes_every_request(line: &str) -> bool { + line.starts_with(BILLING_HEADER_PREFIX) + || (line.starts_with(TOKEN_BUDGET_OPEN) && line.ends_with(TOKEN_BUDGET_CLOSE)) +} + fn response_messages(values: &[Value]) -> Vec { let mut items = BTreeMap::new(); let mut anthropic_blocks = BTreeMap::::new(); From 56047fd8f961417268e5ec71bb255d3695928534 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Thu, 8 Oct 2026 05:15:02 -0700 Subject: [PATCH 22/30] perf(search): rank and page sessions in SQL, fragment only returned matches A search for a common word took 10 to 50 seconds on a real database while the full-text match itself took milliseconds. Every step after the match paid for every hit: fragments were built for all hits of the page's sessions before keeping three per session, every hit was loaded to dedupe sessions in Rust, three queries re-ran the match, reading session keys through the FTS table loaded each matching document, and ordering by observation time read through event text that sits before it in the row. One set of expressions now ranks hits by rowid and bm25 alone, reads keys from the FTS content table, and takes observation times from the owner-time covering index when hits are many or from event rows when few, applying the time window after that join. Window functions give each session's match count and strongest event, so sessions are ordered and paged in SQL with the total in the same query, and fragments are built only for the matches returned. On the user's 41,000 events, "entry" goes from 13.8 to 0.24 seconds and "the" from 36.9 to 0.79, with identical responses across a 21-query benchmark, which is added under scripts/bench. Co-Authored-By: Claude Opus 5.5 --- .../src/storage/sqlite/search.rs | 479 ++++++++++++++---- scripts/bench/README.md | 18 + scripts/bench/search_bench.py | 160 ++++++ 3 files changed, 552 insertions(+), 105 deletions(-) create mode 100644 scripts/bench/README.md create mode 100644 scripts/bench/search_bench.py diff --git a/crates/abyss-backend/src/storage/sqlite/search.rs b/crates/abyss-backend/src/storage/sqlite/search.rs index f5d1e56..7f68cca 100644 --- a/crates/abyss-backend/src/storage/sqlite/search.rs +++ b/crates/abyss-backend/src/storage/sqlite/search.rs @@ -33,20 +33,31 @@ pub(super) fn session_search( query: SessionSearchQuery, ) -> Result { let query = query.validate()?; - let source = MatchSource::new(&query, user_id); - let total_sessions = count_sessions(connection, &source)?; - let session_ids = load_session_page(connection, &query, &source)?; - let matches = load_matches(connection, &session_ids, &source)?; + let hits = count_hits(connection, &query)?; + search(connection, user_id, query, hits) +} + +/// Runs a validated search; `hits` only chooses how observation times are read. +fn search( + connection: &mut SqliteConnection, + user_id: Uuid, + query: ValidatedSearchQuery, + hits: i64, +) -> Result { + let source = MatchSource::new(&query, user_id, hits); + let (page, total_sessions) = load_session_page(connection, &query, &source)?; + let session_ids = page + .iter() + .map(|(session_pk, _)| *session_pk) + .collect::>(); + let mut matches = load_matches(connection, &session_ids, &source)?; let mut details = load_session_details(connection, user_id, &session_ids)?; - let items = session_ids + let items = page .into_iter() - .filter_map(|session_pk| { + .filter_map(|(session_pk, match_count)| { let details = details.remove(&session_pk)?; - let session_matches = matches.get(&session_pk).cloned().unwrap_or_default(); - let match_count = session_matches - .first() - .map_or(0, |matched| matched.match_count); + let session_matches = matches.remove(&session_pk).unwrap_or_default(); let mut providers = session_matches .iter() .map(|matched| matched.llm_provider.clone()) @@ -91,47 +102,140 @@ pub(super) fn session_search( }) } +/// Matches the strongest events rank first: the lowest bm25, then the latest +/// observation, then the event key, so results are deterministic. +const MATCHES_PER_SESSION: i64 = 3; + #[derive(Clone)] enum FtsBind { Text(String), BigInt(i64), } +/// Above this many FTS hits, observation times come from the owner-time +/// covering index for all of the owner's events at once; below it, each hit +/// reads its own event row, which is cheaper than building that index. +const ROW_LOOKUP_MAX_HITS: i64 = 256; + +/// Where a page's session list is inserted into the shared expressions. +const SESSION_FILTER: &str = "/* session filter */"; + +/// Common table expressions that list every matching event of the owner with +/// its rank, observation time, and position and count within its session. +/// +/// Every step avoids reading wide rows: `hits` takes only rowids and bm25 from +/// the FTS index; event and session keys come from FTS5's content table rather +/// than through the virtual table, which would load each matching document. +/// Observation times sit after the event text in its row, so for many hits +/// they come from the owner-time covering index instead, and the time window +/// is applied after the join so SQLite keeps an index on that list. Event rows +/// are otherwise joined only for filters whose columns precede the text. struct MatchSource { sql: String, binds: Vec, } impl MatchSource { - fn new(query: &ValidatedSearchQuery, user_id: Uuid) -> Self { - let mut sql = String::from( - "FROM usage_events_fts - INNER JOIN llm_usage_events e ON e.id = usage_events_fts.event_pk - WHERE usage_events_fts MATCH ? AND e.user_id = ?", - ); - let mut binds = vec![ - FtsBind::Text(fts_query(&query.text)), - FtsBind::Text(user_id.to_string()), - ]; + fn new(query: &ValidatedSearchQuery, user_id: Uuid, hits: i64) -> Self { + let mut binds = vec![FtsBind::Text(fts_query(&query.text))]; + let read_rows = hits <= ROW_LOOKUP_MAX_HITS; + let (events_cte, joins, time_column) = if read_rows { + ( + "", + "INNER JOIN llm_usage_events e ON e.id = c.c0", + "e.observed_at", + ) + } else { + binds.push(FtsBind::Text(user_id.to_string())); + ( + ", + events AS MATERIALIZED ( + SELECT id, observed_at FROM llm_usage_events WHERE user_id = ? + )", + "INNER JOIN events ev ON ev.id = c.c0", + "ev.observed_at", + ) + }; + let mut conditions = String::from("1 = 1"); + if read_rows { + conditions.push_str(" AND e.user_id = ?"); + binds.push(FtsBind::Text(user_id.to_string())); + } if let Some(from) = query.from { - sql.push_str(" AND e.observed_at >= ?"); + conditions.push_str(" AND "); + conditions.push_str(time_column); + conditions.push_str(" >= ?"); binds.push(FtsBind::BigInt(timestamp_to_micros(from))); } if let Some(to) = query.to { - sql.push_str(" AND e.observed_at < ?"); + conditions.push_str(" AND "); + conditions.push_str(time_column); + conditions.push_str(" < ?"); binds.push(FtsBind::BigInt(timestamp_to_micros(to))); } - push_any_filter(&mut sql, &mut binds, "e.agent_name", &query.agent_names); - push_any_filter(&mut sql, &mut binds, "e.llm_provider", &query.llm_providers); - push_any_filter(&mut sql, &mut binds, "e.llm_model", &query.llm_models); + let mut filters = String::new(); + push_any_filter(&mut filters, &mut binds, "e.agent_name", &query.agent_names); + push_any_filter( + &mut filters, + &mut binds, + "e.llm_provider", + &query.llm_providers, + ); + push_any_filter(&mut filters, &mut binds, "e.llm_model", &query.llm_models); push_exact_filter( - &mut sql, + &mut filters, &mut binds, "e.event_type", query.event_type.as_deref(), ); + let event_join = if read_rows || filters.is_empty() { + "" + } else { + "INNER JOIN llm_usage_events e ON e.id = c.c0" + }; + let sql = format!( + "WITH hits AS MATERIALIZED ( + SELECT rowid AS fts_rowid, bm25(usage_events_fts) AS rank + FROM usage_events_fts + WHERE usage_events_fts MATCH ? + ){events_cte}, + matched AS MATERIALIZED ( + SELECT c.c0 AS event_pk, c.c2 AS session_pk, h.fts_rowid, h.rank, + {time_column} AS observed_at + FROM hits h + INNER JOIN usage_events_fts_content c ON c.id = h.fts_rowid + {joins} + {event_join} + WHERE {conditions}{filters} {SESSION_FILTER} + ), + ranked AS ( + SELECT matched.*, + ROW_NUMBER() OVER ( + PARTITION BY session_pk + ORDER BY rank ASC, observed_at DESC, event_pk ASC + ) AS position, + COUNT(*) OVER (PARTITION BY session_pk) AS match_count + FROM matched + )" + ); Self { sql, binds } } + + /// The expressions with matches limited to `session_ids`, or all sessions. + fn sql_for(&self, session_ids: &[Uuid]) -> (String, Vec) { + let mut binds = self.binds.clone(); + let session_filter = if session_ids.is_empty() { + String::new() + } else { + binds.extend( + session_ids + .iter() + .map(|session_pk| FtsBind::Text(session_pk.to_string())), + ); + format!("AND c.c2 IN ({})", vec!["?"; session_ids.len()].join(", ")) + }; + (self.sql.replace(SESSION_FILTER, &session_filter), binds) + } } fn push_exact_filter( @@ -178,62 +282,80 @@ struct CountRow { count: i64, } -fn count_sessions( +/// FTS hits across all owners, before any filter; reads only the index. +fn count_hits( connection: &mut SqliteConnection, - source: &MatchSource, -) -> Result { - let sql = format!( - "SELECT COUNT(DISTINCT e.session_pk) AS count {}", - source.sql - ); - let count = bound_query(sql, source.binds.clone()) - .get_result::(connection)? - .count; - u64::try_from(count.max(0)) - .map_err(|error| AppError::internal(format!("invalid FTS session count: {error}"))) + query: &ValidatedSearchQuery, +) -> Result { + Ok(bound_query( + "SELECT COUNT(*) AS count FROM usage_events_fts WHERE usage_events_fts MATCH ?".to_owned(), + vec![FtsBind::Text(fts_query(&query.text))], + ) + .get_result::(connection)? + .count) } #[derive(QueryableByName)] -struct SessionIdRow { +struct SessionPageRow { #[diesel(sql_type = Text)] session_pk: String, + #[diesel(sql_type = BigInt)] + match_count: i64, + #[diesel(sql_type = BigInt)] + total_sessions: i64, } +/// One page of sessions with their match counts, ordered by each session's +/// strongest event, and the number of matching sessions. fn load_session_page( connection: &mut SqliteConnection, query: &ValidatedSearchQuery, source: &MatchSource, -) -> Result, AppError> { - let sql = format!( - "SELECT e.session_pk AS session_pk - {} - ORDER BY bm25(usage_events_fts) ASC, e.observed_at DESC, e.session_pk ASC", - source.sql +) -> Result<(Vec<(Uuid, u64)>, u64), AppError> { + let (sql, mut binds) = source.sql_for(&[]); + let page_sql = format!( + "{sql} + SELECT session_pk, match_count, COUNT(*) OVER () AS total_sessions + FROM ranked + WHERE position = 1 + ORDER BY rank ASC, observed_at DESC, session_pk ASC + LIMIT ? OFFSET ?" ); - let rows = bound_query(sql, source.binds.clone()).load::(connection)?; - let page_start = usize::try_from(query.offset) - .map_err(|error| AppError::internal(format!("invalid FTS page offset: {error}")))?; - let page_size = usize::try_from(query.page_size) - .map_err(|error| AppError::internal(format!("invalid FTS page size: {error}")))?; - let page_end = page_start.saturating_add(page_size); - let mut seen = HashSet::new(); - let mut sessions = Vec::with_capacity(page_size); - for row in rows { - let session_pk = parse_uuid(&row.session_pk, "FTS session id")?; - if !seen.insert(session_pk) { - continue; + binds.push(FtsBind::BigInt(i64::from(query.page_size))); + binds.push(FtsBind::BigInt(i64::from(query.offset))); + let rows = bound_query(page_sql, binds).load::(connection)?; + let total_sessions = match rows.first() { + Some(row) => row.total_sessions, + // A page past the end has no rows to carry the total. + None if query.offset > 0 => { + let (sql, binds) = source.sql_for(&[]); + bound_query( + format!("{sql} SELECT COUNT(*) AS count FROM ranked WHERE position = 1"), + binds, + ) + .get_result::(connection)? + .count } - let position = seen.len().saturating_sub(1); - if position >= page_start { - sessions.push(session_pk); - } - if seen.len() >= page_end { - break; - } - } - Ok(sessions) + None => 0, + }; + let page = rows + .into_iter() + .map(|row| { + Ok(( + parse_uuid(&row.session_pk, "FTS session id")?, + u64::try_from(row.match_count.max(0)).map_err(|error| { + AppError::internal(format!("invalid FTS match count: {error}")) + })?, + )) + }) + .collect::, AppError>>()?; + let total_sessions = u64::try_from(total_sessions.max(0)) + .map_err(|error| AppError::internal(format!("invalid FTS session count: {error}")))?; + Ok((page, total_sessions)) } +/// The strongest matches of each session, with highlighted fragments built +/// only for the events returned. fn load_matches( connection: &mut SqliteConnection, session_ids: &[Uuid], @@ -242,19 +364,21 @@ fn load_matches( if session_ids.is_empty() { return Ok(HashMap::new()); } - let selected_placeholders = std::iter::repeat_n("?", session_ids.len()) - .collect::>() - .join(","); - let sql = format!( - "SELECT - e.id AS event_pk, - e.session_pk AS session_pk, + let (sql, mut binds) = source.sql_for(session_ids); + let matches_sql = format!( + "{sql}, + top AS MATERIALIZED ( + SELECT * FROM ranked WHERE position <= {MATCHES_PER_SESSION} + ) + SELECT + top.event_pk AS event_pk, + top.session_pk AS session_pk, e.turn_pk AS turn_pk, e.turn_index AS turn_index, e.event_type AS event_type, e.llm_provider AS llm_provider, e.llm_model AS llm_model, - e.observed_at AS observed_at, + top.observed_at AS observed_at, snippet(usage_events_fts, 4, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS content_fragment, snippet(usage_events_fts, 7, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) @@ -265,38 +389,23 @@ fn load_matches( AS tool_name_fragment, snippet(usage_events_fts, 6, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS tool_content_fragment - {} - AND e.session_pk IN ({selected_placeholders}) - ORDER BY bm25(usage_events_fts) ASC, e.observed_at DESC, e.id ASC", - source.sql - ); - let mut binds = source.binds.clone(); - binds.extend( - session_ids - .iter() - .map(|session_pk| FtsBind::Text(session_pk.to_string())), + FROM top + INNER JOIN usage_events_fts ON usage_events_fts.rowid = top.fts_rowid + INNER JOIN llm_usage_events e ON e.id = top.event_pk + WHERE usage_events_fts MATCH ? + ORDER BY top.session_pk, top.position" ); - let rows = bound_query(sql, binds).load::(connection)?; - let mut by_session: HashMap)> = HashMap::new(); + binds.push(source.binds[0].clone()); + let rows = bound_query(matches_sql, binds).load::(connection)?; + let mut by_session: HashMap> = HashMap::new(); for row in rows { let matched = row.into_matched()?; - let entry = by_session + by_session .entry(matched.session_pk) - .or_insert_with(|| (0, Vec::new())); - entry.0 = entry.0.saturating_add(1); - if entry.1.len() < 3 { - entry.1.push(matched); - } + .or_default() + .push(matched); } - Ok(by_session - .into_iter() - .map(|(session_pk, (match_count, mut matches))| { - for matched in &mut matches { - matched.match_count = match_count; - } - (session_pk, matches) - }) - .collect()) + Ok(by_session) } #[derive(QueryableByName)] @@ -352,13 +461,11 @@ impl MatchedEventRow { llm_provider: self.llm_provider, llm_model: self.llm_model, observed_at: timestamp_from_micros(self.observed_at)?, - match_count: 0, fragments, }) } } -#[derive(Clone)] struct MatchedEvent { event_pk: Uuid, session_pk: Uuid, @@ -368,7 +475,6 @@ struct MatchedEvent { llm_provider: String, llm_model: String, observed_at: DateTime, - match_count: u64, fragments: Vec, } @@ -473,7 +579,21 @@ fn fts_query(value: &str) -> String { #[cfg(test)] mod tests { - use super::fts_query; + use diesel::{Connection, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use crate::{ + search::{SessionSearchQuery, SessionSearchResponse}, + usage::IngestEventsRequest, + }; + + use super::{ + super::{migrations, repository::ingest_events}, + ROW_LOOKUP_MAX_HITS, fts_query, search, + }; + + const OWNER_ID: Uuid = Uuid::from_u128(1); #[test] fn fts_query_uses_literal_and_terms() { @@ -483,4 +603,153 @@ mod tests { ); assert_eq!(fts_query(r#"quoted"value"#), r#""quoted""value""#); } + + fn event(id: &str, session: &str, observed_at: &str, text: &str) -> Value { + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos"}, + "agent": {"name": "codex"}, + "session_id": session, + "turn_index": 1, + "llm": {"provider": "openai", "model": "gpt-5"}, + "event_type": "response", + "text": text, + "token_usage": {"output_tokens": 1}, + "metadata": {} + }) + } + + /// Three sessions: `alpha` has five equal hits, `beta` one strong hit, and + /// `gamma` one hit diluted by other words. + fn seeded() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + let mut events = (1_u32..=5_u32) + .map(|minute| { + event( + &format!("a{minute}"), + "alpha", + &format!("2026-09-01T00:0{minute}:00Z"), + "needle note", + ) + }) + .collect::>(); + events.push(event("b1", "beta", "2026-09-02T00:00:00Z", "needle")); + events.push(event( + "g1", + "gamma", + "2026-09-03T00:00:00Z", + "needle among many other unrelated words in a much longer reply", + )); + events.push(event("z1", "zeta", "2026-09-04T00:00:00Z", "nothing here")); + let request: IngestEventsRequest = + serde_json::from_value(json!({ "events": events })).expect("request should parse"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn run(database: &mut SqliteConnection, query: Value, hits: i64) -> SessionSearchResponse { + let query: SessionSearchQuery = + serde_json::from_value(query).expect("query should deserialize"); + search( + database, + OWNER_ID, + query.validate().expect("query should validate"), + hits, + ) + .expect("search should succeed") + } + + fn summary(response: &SessionSearchResponse) -> Vec<(String, u64, usize)> { + response + .items + .iter() + .map(|item| { + ( + item.session_id.clone(), + item.match_count, + item.matches.len(), + ) + }) + .collect() + } + + #[test] + fn sessions_rank_by_their_strongest_event_and_page_in_order() { + let mut database = seeded(); + + let first = run(&mut database, json!({"q": "needle", "page_size": 2_u32}), 0); + let second = run( + &mut database, + json!({"q": "needle", "page": 2_u32, "page_size": 2_u32}), + 0, + ); + let past_end = run( + &mut database, + json!({"q": "needle", "page": 5_u32, "page_size": 2_u32}), + 0, + ); + + assert_eq!(first.total_sessions, 3); + assert!(first.has_more); + assert_eq!( + summary(&first), + [("beta".to_owned(), 1, 1), ("alpha".to_owned(), 5, 3)] + ); + assert_eq!(summary(&second), [("gamma".to_owned(), 1, 1)]); + assert!(!second.has_more); + assert_eq!(past_end.total_sessions, 3); + assert!(past_end.items.is_empty()); + } + + #[test] + fn a_session_returns_its_three_latest_equal_matches_and_counts_all() { + let mut database = seeded(); + + let response = run(&mut database, json!({"q": "needle note"}), 0); + + assert_eq!(summary(&response), [("alpha".to_owned(), 5, 3)]); + let minutes = response.items[0] + .matches + .iter() + .map(|matched| matched.observed_at.format("%M").to_string()) + .collect::>(); + // Equal scores fall back to the latest observation first. + assert_eq!(minutes, ["05", "04", "03"]); + assert!( + response.items[0].matches[0].fragments[0] + .segments + .iter() + .any(|segment| segment.highlighted) + ); + } + + #[test] + fn both_time_lookups_return_the_same_results_with_filters() { + let mut database = seeded(); + for query in [ + json!({"q": "needle"}), + json!({"q": "needle", "from": "2026-09-01T00:03:00Z", "to": "2026-09-02T12:00:00Z"}), + json!({"q": "needle", "agent_name": ["codex"], "event_type": "response", "page_size": 1_u32}), + json!({"q": "needle", "llm_model": ["absent"]}), + ] { + let by_rows = run(&mut database, query.clone(), 0); + let by_index = run(&mut database, query.clone(), ROW_LOOKUP_MAX_HITS + 1); + assert_eq!( + serde_json::to_value(&by_rows).expect("serializes"), + serde_json::to_value(&by_index).expect("serializes"), + "{query}" + ); + } + let windowed = run( + &mut database, + json!({"q": "needle", "from": "2026-09-01T00:03:00Z", "to": "2026-09-02T12:00:00Z"}), + ROW_LOOKUP_MAX_HITS + 1, + ); + assert_eq!( + summary(&windowed), + [("beta".to_owned(), 1, 1), ("alpha".to_owned(), 3, 3)] + ); + } } diff --git a/scripts/bench/README.md b/scripts/bench/README.md new file mode 100644 index 0000000..92cad64 --- /dev/null +++ b/scripts/bench/README.md @@ -0,0 +1,18 @@ +# Search benchmark + +`search_bench.py` times `GET /v1/agent-usage/search` on a running backend +against a set of queries covering missing, rare, Chinese, common, multi-term, +paged, and filtered searches. It records the cold first run, the median of +three warm runs, the session total, and a hash of the canonical response, so +two builds on the same database can be compared for speed and identical +results: + +```bash +python3 scripts/bench/search_bench.py --base-url http://127.0.0.1:PORT \ + --token-file TOKEN --out before.json +python3 scripts/bench/search_bench.py --compare before.json after.json +python3 scripts/bench/search_bench.py --phases DB.sqlite --term entry +``` + +`--phases` times the original three SQL phases directly against a database +copy. Run benchmarks against a copy, never a live database. diff --git a/scripts/bench/search_bench.py b/scripts/bench/search_bench.py new file mode 100644 index 0000000..b0efc1a --- /dev/null +++ b/scripts/bench/search_bench.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Benchmark GET /v1/agent-usage/search against a running backend. + +The script starts nothing. Point it at a backend with --base-url and +--token-file; it runs every query in the set, records the first (cold) and +median warm latency, the session total, and a hash of the canonical response +body so two runs can be compared for identical results. + + python3 scripts/bench/search_bench.py \ + --base-url http://127.0.0.1:53792 --token-file run/token \ + --out run/baseline.json + python3 scripts/bench/search_bench.py --compare run/baseline.json run/after.json + +--phases DB times the three SQL phases of the original implementation with +Python's sqlite3 against a database copy, to attribute where time goes. +""" + +import argparse +import hashlib +import json +import sqlite3 +import statistics +import sys +import time +import urllib.parse +import urllib.request + +# Terms chosen from real records: hit counts range from none to ~24k events. +QUERIES = [ + {"name": "missing", "q": "qqvvxxnone7"}, + {"name": "rare", "q": "bm25"}, + {"name": "chinese_run", "q": "帮你尝试不同的思路"}, + {"name": "medium_timeline", "q": "timeline"}, + {"name": "medium_useQuery", "q": "useQuery"}, + {"name": "common_entry", "q": "entry"}, + {"name": "common_dashboard", "q": "dashboard"}, + {"name": "very_common_the", "q": "the"}, + {"name": "and_two_terms", "q": "cargo sqlite"}, + {"name": "and_three_terms", "q": "git status worktree"}, + {"name": "entry_page2", "q": "entry", "page": 2}, + {"name": "entry_page5", "q": "entry", "page": 5}, + {"name": "entry_page_size50", "q": "entry", "page_size": 50}, + {"name": "entry_past_end", "q": "entry", "page": 40}, + {"name": "entry_codex", "q": "entry", "agent_name": ["codex"]}, + {"name": "entry_two_agents", "q": "entry", "agent_name": ["codex", "claude-code"]}, + {"name": "entry_model", "q": "entry", "llm_model": ["claude-opus-5-5"]}, + {"name": "entry_requests", "q": "entry", "event_type": "request"}, + {"name": "entry_responses", "q": "entry", "event_type": "response"}, + {"name": "entry_week", "q": "entry", "from": "2026-10-01T07:00:00Z", "to": "2026-10-08T07:00:00Z"}, + {"name": "the_week_codex", "q": "the", "from": "2026-10-01T07:00:00Z", "agent_name": ["codex"]}, +] + + +def request(base_url, token, query): + params = [] + for key, value in query.items(): + if key == "name": + continue + for item in value if isinstance(value, list) else [value]: + params.append((key, str(item))) + url = f"{base_url}/v1/agent-usage/search?{urllib.parse.urlencode(params)}" + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + started = time.perf_counter() + with urllib.request.urlopen(req, timeout=600) as response: + body = response.read() + return time.perf_counter() - started, body + + +def canonical_hash(body): + payload = json.loads(body) + return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def run(args): + token = open(args.token_file, encoding="utf-8").read().strip() + results = [] + for query in QUERIES: + cold, body = request(args.base_url, token, query) + warm = [] + for _ in range(args.repeat): + elapsed, warm_body = request(args.base_url, token, query) + warm.append(elapsed) + if canonical_hash(warm_body) != canonical_hash(body): + print(f"warning: {query['name']} answered differently between runs", file=sys.stderr) + payload = json.loads(body) + result = { + "name": query["name"], + "query": {k: v for k, v in query.items() if k != "name"}, + "cold_s": round(cold, 4), + "median_s": round(statistics.median(warm), 4), + "total_sessions": payload["total_sessions"], + "items": len(payload["items"]), + "hash": canonical_hash(body), + } + results.append(result) + print(f"{result['name']:20} cold {cold:7.3f}s median {result['median_s']:7.3f}s " + f"sessions {result['total_sessions']:5} items {result['items']}") + with open(args.out, "w", encoding="utf-8") as handle: + json.dump(results, handle, ensure_ascii=False, indent=2) + + +def compare(before_path, after_path): + before = {r["name"]: r for r in json.load(open(before_path, encoding="utf-8"))} + after = {r["name"]: r for r in json.load(open(after_path, encoding="utf-8"))} + print(f"{'query':20} {'before':>9} {'after':>9} {'speedup':>8} same") + for name, old in before.items(): + new = after.get(name) + if new is None: + continue + speedup = old["median_s"] / new["median_s"] if new["median_s"] else float("inf") + same = "yes" if old["hash"] == new["hash"] else "NO" + print(f"{name:20} {old['median_s']:8.3f}s {new['median_s']:8.3f}s {speedup:7.1f}x {same}") + + +def phases(db_path, term): + """Times the original three SQL phases for one term.""" + db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + user_id = db.execute("SELECT id FROM app_users LIMIT 1").fetchone()[0] + match = " AND ".join('"' + part.replace('"', '""') + '"' for part in term.split()) + source = ("FROM usage_events_fts INNER JOIN llm_usage_events e ON e.id = usage_events_fts.event_pk " + "WHERE usage_events_fts MATCH ? AND e.user_id = ?") + + def timed(label, sql, params): + started = time.perf_counter() + rows = db.execute(sql, params).fetchall() + print(f"{label:34} {time.perf_counter() - started:7.3f}s rows {len(rows)}") + return rows + + timed("count sessions", f"SELECT COUNT(DISTINCT e.session_pk) {source}", (match, user_id)) + rows = timed("all matches, ordered", f"SELECT e.session_pk {source} ORDER BY bm25(usage_events_fts), " + "e.observed_at DESC, e.session_pk", (match, user_id)) + page = list(dict.fromkeys(row[0] for row in rows))[:20] + marks = ",".join("?" * len(page)) + snippets = ", ".join(f"snippet(usage_events_fts, {column}, '[', ']', '…', 32)" for column in (4, 7, 8, 5, 6)) + timed("page matches with snippets", f"SELECT e.id, {snippets} {source} AND e.session_pk IN ({marks}) " + "ORDER BY bm25(usage_events_fts), e.observed_at DESC, e.id", (match, user_id, *page)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base-url") + parser.add_argument("--token-file") + parser.add_argument("--out") + parser.add_argument("--repeat", type=int, default=3) + parser.add_argument("--compare", nargs=2, metavar=("BEFORE", "AFTER")) + parser.add_argument("--phases", metavar="DB") + parser.add_argument("--term", default="entry") + args = parser.parse_args() + if args.compare: + compare(*args.compare) + elif args.phases: + phases(args.phases, args.term) + elif args.base_url and args.token_file and args.out: + run(args) + else: + parser.error("pass --base-url, --token-file and --out, or --compare, or --phases") + + +if __name__ == "__main__": + main() From 40f15d8bd1480664727dbbe3c5e74ed5b16e119f Mon Sep 17 00:00:00 2001 From: Auceptin Date: Mon, 12 Oct 2026 23:52:58 -0700 Subject: [PATCH 23/30] feat(usage): consume structured AgentEvent fields instead of provider bodies Titles and trajectories read the collector's role-aware `metadata.conversation`; the provider-body decoders are removed, so the backend never parses raw request or response bodies. Turns are numbered by the backend from the traffic's turn identity (`codex_turn_id`, `claude_turn_id`, `turn_id`, then provider ids); the sender's `turn_index` is ignored. A Claude Code event without a turn id joins the turn of a same-session `cc_prompt_id`, else the latest turn started at or before it. Claude `thread` continue requests join the stream of the response they continue. Session titles come from collector fields only: a `session_title` names its own session and outranks the first-user-input default, which never replaces it; `purpose` requests are not user input; Codex `thread_source == "thread_title"` sessions are marked `title_generation`. The Codex link heuristic, text-based naming recognition and their backfills are removed. Stored titles and links are kept; no migration. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 +- .../src/storage/postgres/repository.rs | 163 ++- .../src/storage/postgres/session_titles.rs | 563 ++------ .../src/storage/sqlite/repository.rs | 284 +++- .../src/storage/sqlite/session_titles.rs | 1202 ++++------------- .../abyss-backend/src/usage/conversation.rs | 111 ++ crates/abyss-backend/src/usage/mod.rs | 6 +- crates/abyss-backend/src/usage/persistence.rs | 87 +- .../abyss-backend/src/usage/provider_body.rs | 134 -- .../abyss-backend/src/usage/session_title.rs | 517 ++----- crates/abyss-backend/src/usage/trajectory.rs | 393 +++++- .../src/usage/trajectory/conversation.rs | 229 ++++ .../src/usage/trajectory/raw_body.rs | 245 ---- scripts/tests/blackbox_api.py | 4 + scripts/tests/claude_turns_api.py | 76 ++ scripts/tests/session_titles_api.py | 80 +- 16 files changed, 1674 insertions(+), 2437 deletions(-) create mode 100644 crates/abyss-backend/src/usage/conversation.rs delete mode 100644 crates/abyss-backend/src/usage/provider_body.rs create mode 100644 crates/abyss-backend/src/usage/trajectory/conversation.rs delete mode 100644 crates/abyss-backend/src/usage/trajectory/raw_body.rs create mode 100644 scripts/tests/claude_turns_api.py diff --git a/README.md b/README.md index 7bf6865..0b6d535 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,15 @@ and optional full-text search. - Health and storage-readiness probes. - Embedded migrations for both supported storage profiles. -Session titles have dedicated adapters for Codex and Claude Code. Claude Code -names its own session; Codex names a separate task thread, matched by quoted -user content, turn start time, and device. -Only a unique task candidate receives the generated title. Other Agents use -their first nonempty user input, collapsed to one line and truncated to 80 -characters with an ellipsis. Both adapters read archived protocol messages and support older -structured event records. Auxiliary title calls retain their own events and usage. -Existing sessions receive default titles through a one-time backfill. +A session is titled by the `session_title` the collector reports for a call in +which the Agent names its own session (Claude Code); otherwise by its first +nonempty user input, collapsed to one line and truncated to 80 characters with +an ellipsis. An Agent title is never replaced by a default title. Requests whose +`purpose` marks them as naming calls are not user input. Codex names tasks from +separate threads that do not say which session they name; those threads are +marked `title_generation` from their `thread_source` and are not linked to a +task. Naming calls retain their own events and usage. Existing sessions receive +default titles through a one-time backfill. Exactly one storage profile is compiled into a binary: diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index 391d23a..b26fd12 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -46,11 +46,12 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, TurnIdentityKind, agent_names_filter, canonical_agent_name, - choose_turn_index, escape_like_pattern, exact_values_filter, non_empty, + SessionTurnAttachment, TurnIdentity, TurnIdentityKind, agent_names_filter, + canonical_agent_name, escape_like_pattern, exact_values_filter, non_empty, non_negative_count, normalize_limit, normalize_slug, normalize_text, - normalized_optional, normalized_tokens, parse_group_by, providers_filter, sha256_bytes, - turn_identity, unix_epoch, validate_batch, validate_event_type, + normalized_optional, normalized_tokens, parse_group_by, providers_filter, + session_turn_attachment, sha256_bytes, turn_identity, unix_epoch, validate_batch, + validate_event_type, }, session_metadata_from_event, }, @@ -59,7 +60,7 @@ use crate::{ use super::session_titles; #[cfg(test)] -use crate::usage::persistence::{turn_identity_from_metadata, validate_event}; +use crate::usage::persistence::turn_identity_from_metadata; const SUMMARY_AGGREGATE_SQL: &str = "\ SELECT \ @@ -703,7 +704,7 @@ fn ingest_one_event( now, }, )?; - let turn_index = resolve_turn_index(connection, session.id, event)?; + let turn_index = resolve_turn_index(connection, session.id, event, &agent_name, observed_at)?; let turn = upsert_turn( connection, user_id, @@ -771,10 +772,7 @@ fn ingest_one_event( user_id, session_pk: session.id, event_pk: usage_event_id, - turn_index, - agent_name: &agent_name, event_type: event.event_type.as_str(), - text: event.text.as_deref(), metadata: &event.metadata, observed_at, }, @@ -900,20 +898,76 @@ fn resolve_turn_index( connection: &mut PgConnection, session_pk: Uuid, event: &IngestUsageEvent, + agent_name: &str, + observed_at: DateTime, ) -> Result { - // Older collectors without a stable provider identity retain their supplied - // index. Newer evidence lets restarted collectors avoid merging new work - // into an already-used local turn number. - let Some(identity) = turn_identity(event) else { - return Ok(event.turn_index); - }; - let existing_turn_index = existing_turn_index_for_identity(connection, session_pk, &identity)?; - let next_turn_index = next_session_turn_index(connection, session_pk)?; - Ok(choose_turn_index( - event.turn_index, - existing_turn_index, - next_turn_index, - )) + if let Some(attachment) = session_turn_attachment(agent_name, &event.metadata) + && let Some(turn_index) = + attached_turn_index(connection, session_pk, &attachment, observed_at)? + { + return Ok(turn_index); + } + // Events of one turn share its identity; an event without any identity + // is a turn of its own. + if let Some(identity) = turn_identity(event) + && let Some(existing) = existing_turn_index_for_identity(connection, session_pk, &identity)? + { + return Ok(existing); + } + next_session_turn_index(connection, session_pk) +} + +/// Finds the stored turn a Claude Code side or subagent request belongs to; +/// `None` only when the session has no turn yet. +fn attached_turn_index( + connection: &mut PgConnection, + session_pk: Uuid, + attachment: &SessionTurnAttachment, + observed_at: DateTime, +) -> Result, AppError> { + if let Some(prompt_id) = &attachment.prompt_id { + // Only main-loop events anchor a prompt, and each prompt id belongs to + // one turn. Walking the session index from its latest turn usually + // stops within the current turn instead of detoasting every megabyte + // metadata row of the session. + let prompt_turn = sql_query( + "SELECT turn_index \ + FROM llm_usage_events \ + WHERE session_pk = $1 \ + AND metadata->'claude_code_billing'->>'cc_prompt_id' = $2 \ + AND metadata->>'claude_turn_id' IS NOT NULL \ + ORDER BY turn_index DESC, observed_at DESC \ + LIMIT 1", + ) + .bind::(session_pk) + .bind::(prompt_id) + .get_result::(connection) + .optional()?; + if let Some(row) = prompt_turn { + return Ok(Some(row.turn_index)); + } + } + // Events may arrive out of order, so the observation time picks the turn. + let preceding = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .filter(agent_turns::started_at.le(observed_at)) + .order_by(( + agent_turns::started_at.desc(), + agent_turns::turn_index.desc(), + )) + .select(agent_turns::turn_index) + .first::(connection) + .optional()?; + if preceding.is_some() { + return Ok(preceding); + } + agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .order_by((agent_turns::started_at.asc(), agent_turns::turn_index.asc())) + .select(agent_turns::turn_index) + .first::(connection) + .optional() + .map_err(AppError::from) } fn existing_turn_index_for_identity( @@ -942,6 +996,16 @@ fn existing_turn_index_for_identity( .bind::(session_pk) .bind::(&identity.value) .get_result::(connection), + TurnIdentityKind::TurnId => sql_query( + "SELECT turn_index \ + FROM llm_usage_events \ + WHERE session_pk = $1 AND metadata->>'turn_id' = $2 \ + ORDER BY observed_at ASC \ + LIMIT 1", + ) + .bind::(session_pk) + .bind::(&identity.value) + .get_result::(connection), TurnIdentityKind::ResponseId => sql_query( "SELECT turn_index \ FROM llm_usage_events \ @@ -1452,9 +1516,9 @@ mod tests { use super::{ EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, TurnIdentityKind, - canonical_agent_name, choose_turn_index, event_response, latest_optional, normalize_limit, - normalize_slug, normalized_tokens, parse_group_by, turn_identity_from_metadata, - validate_batch, validate_event, validate_event_type, + canonical_agent_name, event_response, latest_optional, normalize_limit, normalize_slug, + normalized_tokens, parse_group_by, turn_identity_from_metadata, validate_batch, + validate_event_type, }; use chrono::{DateTime, TimeZone, Utc}; use serde_json::json; @@ -1742,18 +1806,6 @@ mod tests { assert_eq!(sql_filters.event_type.as_deref(), Some("response")); } - #[test] - fn validate_event_rejects_zero_turn_index() { - let mut event = sample_event(); - event.turn_index = 0_i32; - - let error = validate_event(&event).expect_err("zero turn_index should be invalid"); - assert!( - matches!(&error, AppError::Validation(message) if message.contains("turn_index")), - "zero turn_index should return a validation error" - ); - } - #[test] fn validate_batch_rejects_oversized_batches() { let request = IngestEventsRequest { @@ -1829,42 +1881,6 @@ mod tests { ); } - #[test] - fn choose_turn_index_reuses_existing_provider_identity() { - assert_eq!( - choose_turn_index(9_i32, Some(1_i32), 5_i32), - 1_i32, - "request and response events for the same provider response should share one turn" - ); - } - - #[test] - fn choose_turn_index_allocates_after_conflicting_restarted_counter() { - assert_eq!( - choose_turn_index(1_i32, None, 5_i32), - 5_i32, - "a restarted collector must not merge a new provider response into an old turn" - ); - } - - #[test] - fn choose_turn_index_allocates_after_restarted_counter_gap() { - assert_eq!( - choose_turn_index(2_i32, None, 5_i32), - 5_i32, - "a restarted collector should append even if earlier filtered turns left a gap" - ); - } - - #[test] - fn choose_turn_index_keeps_requested_gap_when_there_is_no_collision() { - assert_eq!( - choose_turn_index(4_i32, None, 2_i32), - 4_i32, - "non-conflicting collector indexes should remain stable" - ); - } - #[test] fn turn_identity_prefers_codex_turn_id_before_provider_response_id() { let identity = turn_identity_from_metadata(&json!({ @@ -2081,7 +2097,6 @@ mod tests { version: Some("1.0.0".to_owned()), }, session_id: "session-001".to_owned(), - turn_index: 1_i32, llm: LlmPayload { provider: "OpenAI".to_owned(), model: "gpt-5.5".to_owned(), diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index 2311539..77862a4 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -1,13 +1,8 @@ -//! PostgreSQL persistence of session names read from title-generation calls. +//! PostgreSQL persistence of session names reported by the collector. -use std::collections::BTreeSet; - -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use diesel::{ - BoolExpressionMethods, Connection, ExpressionMethods, OptionalExtension, PgConnection, - QueryDsl, RunQueryDsl, TextExpressionMethods, - dsl::sql, - sql_types::{Nullable, Text}, + Connection, ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl, }; use serde_json::Value; use uuid::Uuid; @@ -17,25 +12,12 @@ use crate::{ error::AppError, usage::{ SessionInfo, - session_title::{ - NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, - default_title, may_carry_title, opens_with_prompt, title_from_response, - }, + session_title::{TITLE_GENERATION_PURPOSE, TitleContribution, default_title}, }, }; -/// Marks the one-time scan of events stored before titles were captured. -const BACKFILL_NAME: &str = "session_titles_v1"; -/// Marks the one-time matching of Codex naming threads captured before -/// matching existed. -const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; -/// Rebuild links after switching to archived request bodies and scoped adapters. -const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; -/// Names existing sessions from other Agents using their first user input. +/// Names existing sessions using their first user input. const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; -/// Widest delivery delay tolerated between a naming thread and its task when -/// looking for candidates, in hours. -const LINK_DELIVERY_WINDOW_HOURS: i64 = 24; /// Stored title, purpose, and matched task session of one session. type StoredTitle = (Option, Option, Option); @@ -45,72 +27,57 @@ pub(super) struct CapturedEvent<'a> { pub(super) user_id: Uuid, pub(super) session_pk: Uuid, pub(super) event_pk: Uuid, - pub(super) turn_index: i32, - pub(super) agent_name: &'a str, pub(super) event_type: &'a str, - pub(super) text: Option<&'a str>, pub(super) metadata: &'a Value, pub(super) observed_at: DateTime, } -/// A naming call recognized from one of its events. -struct NamingCall { - task: TitleTask, - /// Title with the response event and observation time that carried it. - title: Option<(String, Uuid, DateTime)>, - /// Task message a Codex naming request quotes. - prompt: Option, -} - -/// A title to record for one session. -struct TitleUpdate<'a> { - session_pk: Uuid, - user_id: Uuid, - title: Option<(String, Uuid, DateTime)>, - purpose: Option<&'a str>, -} - -/// Records the session title when the event completes a naming call, and -/// matches Codex naming threads with the task sessions they name. +/// Records what a newly stored event says about its session's name. /// -/// A naming request and its response arrive as separate events in either -/// order, so each side looks up the other within the same turn. Likewise a -/// naming thread and its task session may arrive in either order, so both -/// sides try the match. +/// An Agent title replaces a default title and an older Agent title. A default +/// title keeps the earliest user input, even after late delivery, and never +/// replaces an Agent title. pub(super) fn capture( connection: &mut PgConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { - if !matches!(event.agent_name, "codex" | "claude-code") { - return capture_default(connection, event); - } - match naming_call(connection, event)? { - Some(call) => store(connection, event, call), - None => match_pending_naming_threads(connection, event), - } -} - -/// Default sessions keep the earliest captured user input, even after late delivery. -fn capture_default( - connection: &mut PgConnection, - event: &CapturedEvent<'_>, -) -> Result<(), AppError> { - if event.event_type != "request" { - return Ok(()); - } let session = event.session_pk; - let stored_at = session_titles::table + let agent_title = match TitleContribution::of(event.event_type, event.metadata) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; + let stored = session_titles::table .find(session) - .select(session_titles::title_observed_at) - .first::>>(connection) + .select(( + session_titles::title_event_pk, + session_titles::title_observed_at, + )) + .first::<(Option, Option>)>(connection) .optional()? - .flatten(); - if stored_at.is_some_and(|at| at <= event.observed_at) { + .and_then(|(event_pk, observed_at)| event_pk.zip(observed_at)); + let agent = agent_title.is_some(); + let replaces = match stored { + None => true, + Some((_, stored_at)) if agent && stored_at < event.observed_at => true, + Some((_, stored_at)) if !agent && stored_at <= event.observed_at => false, + // Otherwise only a stored default title gives way. + Some((event_pk, _)) => !is_agent_title(connection, event_pk)?, + }; + if !replaces { return Ok(()); } - let Some(title) = default_title(event.metadata) else { + // Read the user input only once it could name the session. + let Some(title) = agent_title + .map(str::to_owned) + .or_else(|| default_title(event.metadata)) + else { return Ok(()); }; + let now = Utc::now(); diesel::insert_into(session_titles::table) .values(( session_titles::session_pk.eq(session), @@ -118,7 +85,7 @@ fn capture_default( session_titles::title.eq(&title), session_titles::title_event_pk.eq(event.event_pk), session_titles::title_observed_at.eq(event.observed_at), - session_titles::updated_at.eq(Utc::now()), + session_titles::updated_at.eq(now), )) .on_conflict(session_titles::session_pk) .do_update() @@ -126,333 +93,51 @@ fn capture_default( session_titles::title.eq(&title), session_titles::title_event_pk.eq(event.event_pk), session_titles::title_observed_at.eq(event.observed_at), - session_titles::updated_at.eq(Utc::now()), + session_titles::updated_at.eq(now), )) .execute(connection)?; Ok(()) } -fn naming_call( - connection: &mut PgConnection, - event: &CapturedEvent<'_>, -) -> Result, AppError> { - let session = event.session_pk; - match event.event_type { - "request" => { - let Some(naming) = NamingRequest::parse(event.agent_name, event.text, event.metadata) - else { - return Ok(None); - }; - let response = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(session)) - .filter(llm_usage_events::turn_index.eq(event.turn_index)) - .filter(llm_usage_events::event_type.eq("response")) - .filter(llm_usage_events::observed_at.ge(event.observed_at)) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::text, - llm_usage_events::observed_at, - )) - .first::<(Uuid, Option, DateTime)>(connection) - .optional()?; - let title = response.and_then(|(id, text, observed_at)| { - text.as_deref() - .and_then(title_from_response) - .map(|title| (title, id, observed_at)) - }); - Ok(Some(NamingCall { - task: naming.task, - title, - prompt: naming.prompt, - })) - } - "response" if event.text.is_some_and(may_carry_title) => { - let Some(title) = event.text.and_then(title_from_response) else { - return Ok(None); - }; - let request = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(session)) - .filter(llm_usage_events::turn_index.eq(event.turn_index)) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::observed_at.le(event.observed_at)) - .order_by(llm_usage_events::observed_at.desc()) - .select((llm_usage_events::text, llm_usage_events::metadata)) - .first::<(Option, Value)>(connection) - .optional()?; - Ok(request.and_then(|(text, metadata)| { - NamingRequest::parse(event.agent_name, text.as_deref(), &metadata).map(|naming| { - NamingCall { - task: naming.task, - title: Some((title, event.event_pk, event.observed_at)), - prompt: naming.prompt, - } - }) - })) - } - _ => Ok(None), - } +/// Agent titles are read from responses and default titles from requests, so +/// the side of the event that gave the stored title tells them apart. +fn is_agent_title(connection: &mut PgConnection, event_pk: Uuid) -> Result { + Ok(llm_usage_events::table + .find(event_pk) + .select(llm_usage_events::event_type) + .first::(connection) + .optional()? + .is_some_and(|event_type| event_type == "response")) } -fn store( +/// Marks a session as a thread that names another session. +fn mark_title_thread( connection: &mut PgConnection, - event: &CapturedEvent<'_>, - call: NamingCall, + session: Uuid, + user_id: Uuid, ) -> Result<(), AppError> { - let session = event.session_pk; - let auxiliary = call.task == TitleTask::AuxiliarySession; - put_title( - connection, - TitleUpdate { - session_pk: session, - user_id: event.user_id, - title: call.title, - purpose: auxiliary.then_some(TITLE_GENERATION_PURPOSE), - }, - )?; - if !auxiliary { - return Ok(()); - } - let target = session_titles::table.find(session); - if let Some(turn_id) = codex_turn_id(event.metadata) { - diesel::update(target.filter(session_titles::naming_turn_id.is_null())) - .set(session_titles::naming_turn_id.eq(turn_id)) - .execute(connection)?; - } - if let Some(prompt) = call.prompt { - diesel::update(target.filter(session_titles::naming_prompt.is_null())) - .set(session_titles::naming_prompt.eq(prompt)) - .execute(connection)?; - } - match_naming_thread(connection, session) -} - -/// Inserts or updates a session title; a late or replayed naming call never -/// replaces a newer title. -fn put_title(connection: &mut PgConnection, update: TitleUpdate<'_>) -> Result<(), AppError> { let now = Utc::now(); - let (title, title_event_pk, title_observed_at) = match update.title { - Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), - None => (None, None, None), - }; - let existing = session_titles::table - .find(update.session_pk) - .select((session_titles::title_observed_at, session_titles::purpose)) - .first::<(Option>, Option)>(connection) - .optional()?; - let Some((stored_at, stored_purpose)) = existing else { - diesel::insert_into(session_titles::table) - .values(( - session_titles::session_pk.eq(update.session_pk), - session_titles::user_id.eq(update.user_id), - session_titles::title.eq(title), - session_titles::title_event_pk.eq(title_event_pk), - session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(update.purpose), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - return Ok(()); - }; - let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); - let target = session_titles::table.find(update.session_pk); - if newer { - diesel::update(target) - .set(( - session_titles::title.eq(title), - session_titles::title_event_pk.eq(title_event_pk), - session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(update.purpose.map(str::to_owned).or(stored_purpose)), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - } else if stored_purpose.is_none() && update.purpose.is_some() { - diesel::update(target) - .set(( - session_titles::purpose.eq(update.purpose), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - } - Ok(()) -} - -/// Re-examines Codex naming threads whose turn started near this task -/// request's turn: an unmatched one may now have its task, and a matched one -/// that quotes this request too no longer has a single candidate. -fn match_pending_naming_threads( - connection: &mut PgConnection, - event: &CapturedEvent<'_>, -) -> Result<(), AppError> { - if event.event_type != "request" || event.agent_name != "codex" { - return Ok(()); - } - let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { - return Ok(()); - }; - let nearby = session_titles::table - .filter(session_titles::user_id.eq(event.user_id)) - .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .filter(session_titles::title.is_not_null()) - .filter( - session_titles::parent_session_pk - .is_null() - .or(session_titles::parent_session_pk.ne(event.session_pk)), - ) - .filter(session_titles::naming_turn_id.between(low, high)) - .select((session_titles::session_pk, session_titles::naming_prompt)) - .load::<(Uuid, Option)>(connection)?; - if nearby.is_empty() { - return Ok(()); - } - for (naming_session, prompt) in nearby { - if prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { - // Recompute the complete candidate set before adding or withdrawing - // a link. A request from a different device is not a conflict. - match_naming_thread(connection, naming_session)?; - } - } - Ok(()) -} - -/// Withdraws a match and the title it gave, because a second task session -/// quotes the same message. -fn unmatch( - connection: &mut PgConnection, - naming_session: Uuid, - task_session: Uuid, - title_event_pk: Option, -) -> Result<(), AppError> { - diesel::update(session_titles::table.find(naming_session)) - .set(session_titles::parent_session_pk.eq(None::)) + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(user_id), + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) .execute(connection)?; - diesel::update( - session_titles::table - .find(task_session) - .filter(session_titles::title_event_pk.eq(title_event_pk)), - ) - .set(( - session_titles::title.eq(None::), - session_titles::title_event_pk.eq(None::), - session_titles::title_observed_at.eq(None::>), - session_titles::updated_at.eq(Utc::now()), - )) - .execute(connection)?; Ok(()) } -/// Gives a Codex naming thread's title to the one task session whose turn -/// started near the naming turn with the quoted message. Zero or several -/// candidates leave the naming thread unmatched. -fn match_naming_thread( - connection: &mut PgConnection, - naming_session: Uuid, -) -> Result<(), AppError> { - let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt, old_parent)) = - session_titles::table - .find(naming_session) - .select(( - session_titles::user_id, - session_titles::title, - session_titles::title_event_pk, - session_titles::title_observed_at, - session_titles::naming_turn_id, - session_titles::naming_prompt, - session_titles::parent_session_pk, - )) - .first::<( - Uuid, - Option, - Option, - Option>, - Option, - Option, - Option, - )>(connection) - .optional()? - else { - return Ok(()); - }; - let (Some(title), Some(title_event_pk), Some(title_observed_at), Some(prompt)) = - (title, title_event_pk, title_observed_at, prompt) - else { - return Ok(()); - }; - let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { - return Ok(()); - }; - let window = Duration::hours(LINK_DELIVERY_WINDOW_HOURS); - let (Some(earliest), Some(latest)) = ( - title_observed_at.checked_sub_signed(window), - title_observed_at.checked_add_signed(window), - ) else { - return Ok(()); - }; - let device = agent_sessions::table - .find(naming_session) - .select(agent_sessions::device_context_id) - .first::(connection)?; - let candidates = llm_usage_events::table - .filter(llm_usage_events::user_id.eq(user_id)) - .filter(llm_usage_events::observed_at.between(earliest, latest)) - .filter(llm_usage_events::device_context_id.eq(device)) - .filter(llm_usage_events::agent_name.eq("codex")) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::session_pk.ne(naming_session)) - .filter( - sql::>("llm_usage_events.metadata ->> 'codex_turn_id'") - .between(low, high), - ) - .select((llm_usage_events::session_pk, llm_usage_events::metadata)) - .load::<(Uuid, Value)>(connection)?; - let sessions = candidates - .into_iter() - .filter(|(_, metadata)| opens_with_prompt(metadata, &prompt)) - .map(|(session, _)| session) - .collect::>(); - let mut sessions = sessions.into_iter(); - let task_session = match (sessions.next(), sessions.next()) { - (Some(session), None) => Some(session), - _ => None, - }; - if let Some(parent) = old_parent - && Some(parent) != task_session - { - unmatch(connection, naming_session, parent, Some(title_event_pk))?; - } - let Some(task_session) = task_session else { - return Ok(()); - }; - diesel::update(session_titles::table.find(naming_session)) - .set(session_titles::parent_session_pk.eq(task_session)) - .execute(connection)?; - put_title( - connection, - TitleUpdate { - session_pk: task_session, - user_id, - title: Some((title, title_event_pk, title_observed_at)), - purpose: None, - }, - ) -} - -/// Runs the one-time scans that bring events stored before capture existed up -/// to date. +/// Runs the one-time scan that names sessions stored before titles existed. pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { - run_once(connection, BACKFILL_NAME, backfill_titles)?; - run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; - run_once(connection, LINK_REPAIR_NAME, backfill_links)?; - run_once(connection, DEFAULT_BACKFILL_NAME, backfill_defaults) -} - -fn run_once( - connection: &mut PgConnection, - name: &str, - scan: fn(&mut PgConnection) -> Result<(), AppError>, -) -> Result<(), AppError> { let done = schema_backfills::table - .find(name) + .find(DEFAULT_BACKFILL_NAME) .select(schema_backfills::name) .first::(connection) .optional()? @@ -461,10 +146,10 @@ fn run_once( return Ok(()); } connection.transaction(|transaction| { - scan(transaction)?; + backfill_defaults(transaction)?; diesel::insert_into(schema_backfills::table) .values(( - schema_backfills::name.eq(name), + schema_backfills::name.eq(DEFAULT_BACKFILL_NAME), schema_backfills::completed_at.eq(Utc::now()), )) .execute(transaction)?; @@ -472,101 +157,12 @@ fn run_once( }) } -/// Captures titles from naming requests stored before capture existed. -fn backfill_titles(connection: &mut PgConnection) -> Result<(), AppError> { - let requests = llm_usage_events::table - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::agent_name.eq_any(["codex", "claude-code"])) - .filter( - llm_usage_events::text - .like("%You are naming a coding session%") - .or(llm_usage_events::text.like("%Generate a concise, single-line task title%")), - ) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::user_id, - llm_usage_events::session_pk, - llm_usage_events::turn_index, - llm_usage_events::agent_name, - llm_usage_events::text, - llm_usage_events::metadata, - llm_usage_events::observed_at, - )) - .load::<( - Uuid, - Uuid, - Uuid, - i32, - String, - Option, - Value, - DateTime, - )>(connection)?; - for (event_pk, user_id, session_pk, turn_index, agent_name, text, metadata, observed_at) in - requests - { - capture( - connection, - &CapturedEvent { - user_id, - session_pk, - event_pk, - turn_index, - agent_name: &agent_name, - event_type: "request", - text: text.as_deref(), - metadata: &metadata, - observed_at, - }, - )?; - } - Ok(()) -} - -/// Matches naming threads whose titles were captured before matching existed. -fn backfill_links(connection: &mut PgConnection) -> Result<(), AppError> { - let naming_sessions = session_titles::table - .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .select(session_titles::session_pk) - .load::(connection)?; - for naming_session in naming_sessions { - let request = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(naming_session)) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::agent_name.eq("codex")) - .order_by(llm_usage_events::observed_at.desc()) - .select((llm_usage_events::text, llm_usage_events::metadata)) - .first::<(Option, Value)>(connection) - .optional()?; - let Some((text, metadata)) = request else { - continue; - }; - let Some(naming) = NamingRequest::parse("codex", text.as_deref(), &metadata) else { - continue; - }; - diesel::update(session_titles::table.find(naming_session)) - .set(( - session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), - session_titles::naming_prompt.eq(naming.prompt), - )) - .execute(connection)?; - match_naming_thread(connection, naming_session)?; - } - Ok(()) -} - /// Visit requests in time order until the first readable user input is found. fn backfill_defaults(connection: &mut PgConnection) -> Result<(), AppError> { let sessions = agent_sessions::table - .filter(agent_sessions::agent_name.ne_all(["codex", "claude-code"])) - .select(( - agent_sessions::id, - agent_sessions::user_id, - agent_sessions::agent_name, - )) - .load::<(Uuid, Uuid, String)>(connection)?; - for (session, user_id, agent_name) in sessions { + .select((agent_sessions::id, agent_sessions::user_id)) + .load::<(Uuid, Uuid)>(connection)?; + for (session, user_id) in sessions { let mut offset = 0_i64; loop { let requests = llm_usage_events::table @@ -589,19 +185,20 @@ fn backfill_defaults(connection: &mut PgConnection) -> Result<(), AppError> { } let mut found = false; for (event_pk, observed_at, metadata) in requests { - if default_title(&metadata).is_none() { + let user_input = matches!( + TitleContribution::of("request", &metadata), + Some(TitleContribution::UserInput) + ); + if !user_input || default_title(&metadata).is_none() { continue; } - capture_default( + capture( connection, &CapturedEvent { user_id, session_pk: session, event_pk, - turn_index: 0, - agent_name: &agent_name, event_type: "request", - text: None, metadata: &metadata, observed_at, }, diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index e9b9694..e2cc433 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -27,10 +27,11 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, agent_names_filter, canonical_agent_name, choose_turn_index, + SessionTurnAttachment, TurnIdentity, agent_names_filter, canonical_agent_name, exact_values_filter, non_empty, normalize_limit, normalize_slug, normalize_text, - normalized_optional, normalized_tokens, parse_group_by, providers_filter, sha256_bytes, - turn_identity, validate_batch, validate_event_type, + normalized_optional, normalized_tokens, parse_group_by, providers_filter, + session_turn_attachment, sha256_bytes, turn_identity, validate_batch, + validate_event_type, }, session_metadata_from_event, }, @@ -125,7 +126,7 @@ fn ingest_one_event( observed_at, now, )?; - let turn_index = resolve_turn_index(connection, session_pk, event)?; + let turn_index = resolve_turn_index(connection, session_pk, event, &agent_name, observed_at)?; let turn_pk = upsert_turn( connection, user_id, @@ -181,10 +182,7 @@ fn ingest_one_event( user_id, session_pk, event_pk, - turn_index, - agent_name: &agent_name, event_type: event.event_type.as_str(), - text: text.as_deref(), metadata: &event.metadata, observed_at, }, @@ -336,18 +334,87 @@ fn resolve_turn_index( connection: &mut SqliteConnection, session_pk: Uuid, event: &IngestUsageEvent, + agent_name: &str, + observed_at: i64, ) -> Result { - let Some(identity) = turn_identity(event) else { - return Ok(event.turn_index); - }; - let existing = existing_turn_index_for_identity(connection, session_pk, &identity)?; - let next = agent_turns::table + if let Some(attachment) = session_turn_attachment(agent_name, &event.metadata) + && let Some(turn_index) = + attached_turn_index(connection, session_pk, &attachment, observed_at)? + { + return Ok(turn_index); + } + // Events of one turn share its identity; an event without any identity + // is a turn of its own. + if let Some(identity) = turn_identity(event) + && let Some(existing) = existing_turn_index_for_identity(connection, session_pk, &identity)? + { + return Ok(existing); + } + Ok(agent_turns::table .filter(agent_turns::session_pk.eq(session_pk.to_string())) .select(max(agent_turns::turn_index)) .first::>(connection)? .unwrap_or(0_i32) - .saturating_add(1); - Ok(choose_turn_index(event.turn_index, existing, next)) + .saturating_add(1)) +} + +/// Finds the stored turn a Claude Code side or subagent request belongs to; +/// `None` only when the session has no turn yet. +fn attached_turn_index( + connection: &mut SqliteConnection, + session_pk: Uuid, + attachment: &SessionTurnAttachment, + observed_at: i64, +) -> Result, AppError> { + let session = session_pk.to_string(); + if let Some(prompt_id) = &attachment.prompt_id { + // Only main-loop events anchor a prompt, and each prompt id belongs to + // one turn. Walking the session index from its latest turn usually + // stops within the current turn instead of reading every megabyte + // metadata row of the session. + let prompt_turn = sql_query( + "SELECT turn_index FROM llm_usage_events \ + WHERE session_pk = ?1 \ + AND json_extract(metadata, '$.claude_code_billing.cc_prompt_id') = ?2 \ + AND json_extract(metadata, '$.claude_turn_id') IS NOT NULL \ + ORDER BY turn_index DESC, observed_at DESC, id DESC \ + LIMIT 1", + ) + .bind::(&session) + .bind::(prompt_id) + .get_result::(connection) + .optional()?; + if let Some(row) = prompt_turn { + return Ok(Some(row.turn_index)); + } + } + // Events may arrive out of order, so the observation time picks the turn. + let preceding = agent_turns::table + .filter(agent_turns::session_pk.eq(&session)) + .filter(agent_turns::started_at.le(observed_at)) + .order_by(( + agent_turns::started_at.desc(), + agent_turns::turn_index.desc(), + )) + .select(agent_turns::turn_index) + .first::(connection) + .optional()?; + if preceding.is_some() { + return Ok(preceding); + } + agent_turns::table + .filter(agent_turns::session_pk.eq(&session)) + .order_by((agent_turns::started_at.asc(), agent_turns::turn_index.asc())) + .select(agent_turns::turn_index) + .first::(connection) + .optional() + .map_err(AppError::from) +} + +#[derive(QueryableByName)] +struct TurnIndexRow { + #[diesel(sql_type = diesel::sql_types::Integer)] + turn_index: i32, } fn existing_turn_index_for_identity( @@ -1378,7 +1445,10 @@ pub(super) const fn timestamp_to_micros(value: DateTime) -> i64 { #[cfg(test)] mod tests { - use diesel::{Connection, QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star, sql_query}; + use diesel::{ + Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star, + sql_query, + }; use serde_json::json; use uuid::Uuid; @@ -1388,7 +1458,11 @@ mod tests { }; use super::{ - super::{migrations, schema::llm_usage_events, search}, + super::{ + migrations, + schema::{agent_turns, llm_usage_events}, + search, + }, ingest_events, raw_events, session_outline, usage_summary, }; @@ -1579,7 +1653,6 @@ mod tests { "device": {"host_name": "alice-mbp", "platform": "macos", "os_version": "15.1"}, "agent": {"name": agent}, "session_id": session, - "turn_index": turn, "llm": {"provider": provider, "model": model}, "event_type": if id.ends_with("req") { "request" } else { "response" }, "text": "body", @@ -1588,7 +1661,7 @@ mod tests { } else { json!({"output_tokens": 5_i64}) }, - "metadata": {"raw_body": "large"} + "metadata": {"raw_body": "large", "turn_id": format!("{session}-{turn}")} }) }; let request: IngestEventsRequest = serde_json::from_value(json!({ @@ -1719,6 +1792,181 @@ mod tests { assert!(rows[0].get("agent_name").is_none()); } + /// One request of `agent` observed `second` seconds into a fixed minute. + fn turn_event( + id: &str, + agent: &str, + second: u32, + metadata: serde_json::Value, + ) -> serde_json::Value { + let mut event = json!({ + "event_id": id, + "observed_at": format!("2026-10-08T00:00:{second:02}Z"), + "device": {"host_name": "local", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": "thread-session", + // Distinct collector indexes expose any event left on its own turn. + "turn_index": second, + "llm": {"provider": "Anthropic", "model": "claude-x"}, + "event_type": "request", + "token_usage": {"input_tokens": 1_i64} + }); + event["metadata"] = metadata; + event + } + + fn ingest_turn_events(database: &mut SqliteConnection, events: Vec) { + let request: IngestEventsRequest = + serde_json::from_value(json!({"events": serde_json::Value::Array(events)})) + .expect("request should deserialize"); + ingest_events(database, &request, OWNER_ID, 100).expect("ingest should succeed"); + } + + fn turn_of(database: &mut SqliteConnection, event_id: &str) -> i32 { + llm_usage_events::table + .filter(llm_usage_events::event_id.eq(event_id)) + .select(llm_usage_events::turn_index) + .first::(database) + .expect("event should be stored") + } + + fn main_loop(turn: u32, prompt: &str) -> serde_json::Value { + json!({ + "claude_turn_id": format!("cc_turn:{turn}"), + "claude_code_billing": {"cc_prompt_id": prompt, "cc_turn_index": turn.to_string()} + }) + } + + #[test] + fn claude_code_side_and_subagent_requests_join_the_turn_they_occur_in() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![ + turn_event("main-1", "claude-code", 10, main_loop(1, "prompt-1")), + turn_event( + "subagent-1", + "claude-code", + 12, + json!({"claude_code_billing": {"cc_prompt_id": "prompt-1", "cc_is_subagent": "true"}}), + ), + turn_event("main-2", "claude-code", 30, main_loop(2, "prompt-2")), + // A failed `continue` replays with a new prompt id, same turn. + turn_event( + "main-2-replay", + "claude-code", + 35, + main_loop(2, "prompt-2b"), + ), + turn_event( + "subagent-2", + "claude-code", + 40, + json!({"claude_code_billing": {"cc_prompt_id": "prompt-2b", "cc_is_subagent": "true"}}), + ), + // A recap between the turns carries no billing header. + turn_event( + "recap", + "claude-code", + 20, + json!({"message_id": "msg-recap"}), + ), + ], + ); + // Out of order: observed during turn 2, delivered after everything. + ingest_turn_events( + &mut database, + vec![ + turn_event( + "late-title", + "claude-code", + 32, + json!({"request_hash": "title"}), + ), + turn_event( + "late-subagent", + "claude-code", + 45, + json!({"claude_code_billing": {"cc_prompt_id": "prompt-1"}}), + ), + ], + ); + + let main_1 = turn_of(&mut database, "main-1"); + let main_2 = turn_of(&mut database, "main-2"); + assert_ne!(main_1, main_2); + assert_eq!(turn_of(&mut database, "main-2-replay"), main_2); + assert_eq!(turn_of(&mut database, "subagent-1"), main_1); + assert_eq!(turn_of(&mut database, "subagent-2"), main_2); + assert_eq!(turn_of(&mut database, "recap"), main_1); + assert_eq!(turn_of(&mut database, "late-title"), main_2); + assert_eq!(turn_of(&mut database, "late-subagent"), main_1); + let turns = agent_turns::table + .order_by(agent_turns::turn_index.asc()) + .select(( + agent_turns::turn_index, + agent_turns::started_at, + agent_turns::ended_at, + )) + .load::<(i32, i64, Option)>(&mut database) + .expect("turns should load"); + let second = |value: i64| { + super::timestamp_to_micros( + chrono::DateTime::parse_from_rfc3339(&format!("2026-10-08T00:00:{value:02}Z")) + .expect("timestamp should parse") + .into(), + ) + }; + assert_eq!( + turns, + vec![ + (main_1, second(10), Some(second(45))), + (main_2, second(30), Some(second(40))), + ] + ); + } + + #[test] + fn claude_code_side_request_before_any_turn_keeps_todays_turn() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![turn_event( + "title", + "claude-code", + 5, + json!({"request_hash": "title"}), + )], + ); + let first = turn_of(&mut database, "title"); + ingest_turn_events( + &mut database, + vec![ + turn_event("main-1", "claude-code", 10, main_loop(1, "prompt-1")), + // Earlier than every turn: joins the session's earliest turn. + turn_event("probe", "claude-code", 1, json!({})), + ], + ); + assert_ne!(turn_of(&mut database, "main-1"), first); + assert_eq!(turn_of(&mut database, "probe"), first); + } + + #[test] + fn other_agents_keep_provider_turn_identities() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![ + turn_event("codex-1", "codex", 10, json!({"codex_turn_id": "turn-a"})), + turn_event("codex-2", "codex", 20, json!({"response_id": "resp-b"})), + ], + ); + assert_ne!( + turn_of(&mut database, "codex-1"), + turn_of(&mut database, "codex-2") + ); + } + fn request_with_text(text: &str) -> IngestEventsRequest { serde_json::from_value(json!({ "events": [{ diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index 59582fa..c27f56f 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -1,14 +1,7 @@ -//! SQLite persistence of session names read from title-generation calls. - -use std::collections::BTreeSet; +//! SQLite persistence of session names reported by the collector. use chrono::Utc; -use diesel::{ - BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, - SqliteConnection, TextExpressionMethods, - dsl::sql, - sql_types::{Nullable, Text}, -}; +use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; use serde_json::Value; use uuid::Uuid; @@ -20,25 +13,12 @@ use crate::{ error::AppError, usage::{ SessionInfo, - session_title::{ - NamingRequest, TITLE_GENERATION_PURPOSE, TitleTask, codex_turn_id, codex_turn_id_range, - default_title, may_carry_title, opens_with_prompt, title_from_response, - }, + session_title::{TITLE_GENERATION_PURPOSE, TitleContribution, default_title}, }, }; -/// Marks the one-time scan of events stored before titles were captured. -const BACKFILL_NAME: &str = "session_titles_v1"; -/// Marks the one-time matching of Codex naming threads captured before -/// matching existed. -const LINK_BACKFILL_NAME: &str = "codex_title_links_v1"; -/// Rebuild links after switching to archived request bodies and scoped adapters. -const LINK_REPAIR_NAME: &str = "codex_title_links_v2"; -/// Names existing sessions from other Agents using their first user input. +/// Names existing sessions using their first user input. const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; -/// Widest delivery delay tolerated between a naming thread and its task when -/// looking for candidates, in microseconds. -const LINK_DELIVERY_WINDOW_MICROS: i64 = 24 * 60 * 60 * 1_000_000; /// Stored title, purpose, and matched task session of one session. type StoredTitle = (Option, Option, Option); @@ -48,72 +28,57 @@ pub(super) struct CapturedEvent<'a> { pub(super) user_id: Uuid, pub(super) session_pk: Uuid, pub(super) event_pk: Uuid, - pub(super) turn_index: i32, - pub(super) agent_name: &'a str, pub(super) event_type: &'a str, - pub(super) text: Option<&'a str>, pub(super) metadata: &'a Value, pub(super) observed_at: i64, } -/// A naming call recognized from one of its events. -struct NamingCall { - task: TitleTask, - /// Title with the response event and observation time that carried it. - title: Option<(String, String, i64)>, - /// Task message a Codex naming request quotes. - prompt: Option, -} - -/// A title to record for one session. -struct TitleUpdate<'a> { - session_pk: &'a str, - user_id: &'a str, - title: Option<(String, String, i64)>, - purpose: Option<&'a str>, -} - -/// Records the session title when the event completes a naming call, and -/// matches Codex naming threads with the task sessions they name. +/// Records what a newly stored event says about its session's name. /// -/// A naming request and its response arrive as separate events in either -/// order, so each side looks up the other within the same turn. Likewise a -/// naming thread and its task session may arrive in either order, so both -/// sides try the match. +/// An Agent title replaces a default title and an older Agent title. A default +/// title keeps the earliest user input, even after late delivery, and never +/// replaces an Agent title. pub(super) fn capture( connection: &mut SqliteConnection, event: &CapturedEvent<'_>, ) -> Result<(), AppError> { - if !matches!(event.agent_name, "codex" | "claude-code") { - return capture_default(connection, event); - } - match naming_call(connection, event)? { - Some(call) => store(connection, event, call), - None => match_pending_naming_threads(connection, event), - } -} - -/// Default sessions keep the earliest captured user input, even after late delivery. -fn capture_default( - connection: &mut SqliteConnection, - event: &CapturedEvent<'_>, -) -> Result<(), AppError> { - if event.event_type != "request" { - return Ok(()); - } let session = event.session_pk.to_string(); - let stored_at = session_titles::table + let agent_title = match TitleContribution::of(event.event_type, event.metadata) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, &session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; + let stored = session_titles::table .find(&session) - .select(session_titles::title_observed_at) - .first::>(connection) + .select(( + session_titles::title_event_pk, + session_titles::title_observed_at, + )) + .first::<(Option, Option)>(connection) .optional()? - .flatten(); - if stored_at.is_some_and(|at| at <= event.observed_at) { + .and_then(|(event_pk, observed_at)| event_pk.zip(observed_at)); + let agent = agent_title.is_some(); + let replaces = match stored { + None => true, + Some((_, stored_at)) if agent && stored_at < event.observed_at => true, + Some((_, stored_at)) if !agent && stored_at <= event.observed_at => false, + // Otherwise only a stored default title gives way. + Some((event_pk, _)) => !is_agent_title(connection, &event_pk)?, + }; + if !replaces { return Ok(()); } - let Some(title) = default_title(event.metadata) else { + // Read the user input only once it could name the session. + let Some(title) = agent_title + .map(str::to_owned) + .or_else(|| default_title(event.metadata)) + else { return Ok(()); }; + let now = Utc::now().timestamp_micros(); diesel::insert_into(session_titles::table) .values(( session_titles::session_pk.eq(&session), @@ -121,7 +86,7 @@ fn capture_default( session_titles::title.eq(&title), session_titles::title_event_pk.eq(event.event_pk.to_string()), session_titles::title_observed_at.eq(event.observed_at), - session_titles::updated_at.eq(Utc::now().timestamp_micros()), + session_titles::updated_at.eq(now), )) .on_conflict(session_titles::session_pk) .do_update() @@ -129,338 +94,51 @@ fn capture_default( session_titles::title.eq(&title), session_titles::title_event_pk.eq(event.event_pk.to_string()), session_titles::title_observed_at.eq(event.observed_at), - session_titles::updated_at.eq(Utc::now().timestamp_micros()), + session_titles::updated_at.eq(now), )) .execute(connection)?; Ok(()) } -fn naming_call( - connection: &mut SqliteConnection, - event: &CapturedEvent<'_>, -) -> Result, AppError> { - let session = event.session_pk.to_string(); - match event.event_type { - "request" => { - let Some(naming) = NamingRequest::parse(event.agent_name, event.text, event.metadata) - else { - return Ok(None); - }; - let response = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(&session)) - .filter(llm_usage_events::turn_index.eq(event.turn_index)) - .filter(llm_usage_events::event_type.eq("response")) - .filter(llm_usage_events::observed_at.ge(event.observed_at)) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::text, - llm_usage_events::observed_at, - )) - .first::<(String, Option, i64)>(connection) - .optional()?; - let title = response.and_then(|(id, text, observed_at)| { - text.as_deref() - .and_then(title_from_response) - .map(|title| (title, id, observed_at)) - }); - Ok(Some(NamingCall { - task: naming.task, - title, - prompt: naming.prompt, - })) - } - "response" if event.text.is_some_and(may_carry_title) => { - let Some(title) = event.text.and_then(title_from_response) else { - return Ok(None); - }; - let request = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(&session)) - .filter(llm_usage_events::turn_index.eq(event.turn_index)) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::observed_at.le(event.observed_at)) - .order_by(llm_usage_events::observed_at.desc()) - .select((llm_usage_events::text, llm_usage_events::metadata)) - .first::<(Option, String)>(connection) - .optional()?; - Ok(request.and_then(|(text, metadata)| { - let metadata = serde_json::from_str::(&metadata).ok()?; - NamingRequest::parse(event.agent_name, text.as_deref(), &metadata).map(|naming| { - NamingCall { - task: naming.task, - title: Some((title, event.event_pk.to_string(), event.observed_at)), - prompt: naming.prompt, - } - }) - })) - } - _ => Ok(None), - } +/// Agent titles are read from responses and default titles from requests, so +/// the side of the event that gave the stored title tells them apart. +fn is_agent_title(connection: &mut SqliteConnection, event_pk: &str) -> Result { + Ok(llm_usage_events::table + .find(event_pk) + .select(llm_usage_events::event_type) + .first::(connection) + .optional()? + .is_some_and(|event_type| event_type == "response")) } -fn store( +/// Marks a session as a thread that names another session. +fn mark_title_thread( connection: &mut SqliteConnection, - event: &CapturedEvent<'_>, - call: NamingCall, + session: &str, + user_id: Uuid, ) -> Result<(), AppError> { - let session = event.session_pk.to_string(); - let auxiliary = call.task == TitleTask::AuxiliarySession; - put_title( - connection, - TitleUpdate { - session_pk: &session, - user_id: &event.user_id.to_string(), - title: call.title, - purpose: auxiliary.then_some(TITLE_GENERATION_PURPOSE), - }, - )?; - if !auxiliary { - return Ok(()); - } - let target = session_titles::table.find(&session); - if let Some(turn_id) = codex_turn_id(event.metadata) { - diesel::update(target.filter(session_titles::naming_turn_id.is_null())) - .set(session_titles::naming_turn_id.eq(turn_id)) - .execute(connection)?; - } - if let Some(prompt) = call.prompt { - diesel::update(target.filter(session_titles::naming_prompt.is_null())) - .set(session_titles::naming_prompt.eq(prompt)) - .execute(connection)?; - } - match_naming_thread(connection, &session) -} - -/// Inserts or updates a session title; a late or replayed naming call never -/// replaces a newer title. -fn put_title(connection: &mut SqliteConnection, update: TitleUpdate<'_>) -> Result<(), AppError> { let now = Utc::now().timestamp_micros(); - let (title, title_event_pk, title_observed_at) = match update.title { - Some((title, event_pk, observed_at)) => (Some(title), Some(event_pk), Some(observed_at)), - None => (None, None, None), - }; - let existing = session_titles::table - .find(update.session_pk) - .select((session_titles::title_observed_at, session_titles::purpose)) - .first::<(Option, Option)>(connection) - .optional()?; - let Some((stored_at, stored_purpose)) = existing else { - diesel::insert_into(session_titles::table) - .values(( - session_titles::session_pk.eq(update.session_pk), - session_titles::user_id.eq(update.user_id), - session_titles::title.eq(title), - session_titles::title_event_pk.eq(title_event_pk), - session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(update.purpose), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - return Ok(()); - }; - let newer = title.is_some() && stored_at.is_none_or(|stored| title_observed_at > Some(stored)); - let target = session_titles::table.find(update.session_pk); - if newer { - diesel::update(target) - .set(( - session_titles::title.eq(title), - session_titles::title_event_pk.eq(title_event_pk), - session_titles::title_observed_at.eq(title_observed_at), - session_titles::purpose.eq(update.purpose.map(str::to_owned).or(stored_purpose)), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - } else if stored_purpose.is_none() && update.purpose.is_some() { - diesel::update(target) - .set(( - session_titles::purpose.eq(update.purpose), - session_titles::updated_at.eq(now), - )) - .execute(connection)?; - } - Ok(()) -} - -/// Re-examines Codex naming threads whose turn started near this task -/// request's turn: an unmatched one may now have its task, and a matched one -/// that quotes this request too no longer has a single candidate. -fn match_pending_naming_threads( - connection: &mut SqliteConnection, - event: &CapturedEvent<'_>, -) -> Result<(), AppError> { - if event.event_type != "request" || event.agent_name != "codex" { - return Ok(()); - } - let Some((low, high)) = codex_turn_id(event.metadata).and_then(codex_turn_id_range) else { - return Ok(()); - }; - let nearby = session_titles::table - .filter(session_titles::user_id.eq(event.user_id.to_string())) - .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .filter(session_titles::title.is_not_null()) - .filter( - session_titles::parent_session_pk - .is_null() - .or(session_titles::parent_session_pk.ne(event.session_pk.to_string())), - ) - .filter(session_titles::naming_turn_id.between(low, high)) - .select((session_titles::session_pk, session_titles::naming_prompt)) - .load::<(String, Option)>(connection)?; - if nearby.is_empty() { - return Ok(()); - } - for (naming_session, prompt) in nearby { - if prompt.is_some_and(|prompt| opens_with_prompt(event.metadata, &prompt)) { - // Recompute the complete candidate set before adding or withdrawing - // a link. A request from a different device is not a conflict. - match_naming_thread(connection, &naming_session)?; - } - } - Ok(()) -} - -/// Withdraws a match and the title it gave, because a second task session -/// quotes the same message. -fn unmatch( - connection: &mut SqliteConnection, - naming_session: &str, - task_session: &str, - title_event_pk: Option, -) -> Result<(), AppError> { - diesel::update(session_titles::table.find(naming_session)) - .set(session_titles::parent_session_pk.eq(None::)) - .execute(connection)?; - diesel::update( - session_titles::table - .find(task_session) - .filter(session_titles::title_event_pk.eq(title_event_pk)), - ) - .set(( - session_titles::title.eq(None::), - session_titles::title_event_pk.eq(None::), - session_titles::title_observed_at.eq(None::), - session_titles::updated_at.eq(Utc::now().timestamp_micros()), - )) - .execute(connection)?; - Ok(()) -} - -/// Gives a Codex naming thread's title to the one task session whose turn -/// started near the naming turn with the quoted message. Zero or several -/// candidates leave the naming thread unmatched. -fn match_naming_thread( - connection: &mut SqliteConnection, - naming_session: &str, -) -> Result<(), AppError> { - let Some((user_id, title, title_event_pk, title_observed_at, turn_id, prompt, old_parent)) = - session_titles::table - .find(naming_session) - .select(( - session_titles::user_id, - session_titles::title, - session_titles::title_event_pk, - session_titles::title_observed_at, - session_titles::naming_turn_id, - session_titles::naming_prompt, - session_titles::parent_session_pk, - )) - .first::<( - String, - Option, - Option, - Option, - Option, - Option, - Option, - )>(connection) - .optional()? - else { - return Ok(()); - }; - let (Some(title), Some(title_event_pk), Some(title_observed_at), Some(prompt)) = - (title, title_event_pk, title_observed_at, prompt) - else { - return Ok(()); - }; - let Some((low, high)) = turn_id.as_deref().and_then(codex_turn_id_range) else { - return Ok(()); - }; - let device = agent_sessions::table - .find(naming_session) - .select(agent_sessions::device_context_id) - .first::(connection)?; - let candidates = llm_usage_events::table - .filter(llm_usage_events::user_id.eq(&user_id)) - .filter(llm_usage_events::observed_at.between( - title_observed_at.saturating_sub(LINK_DELIVERY_WINDOW_MICROS), - title_observed_at.saturating_add(LINK_DELIVERY_WINDOW_MICROS), + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(user_id.to_string()), + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), )) - .filter(llm_usage_events::device_context_id.eq(&device)) - .filter(llm_usage_events::agent_name.eq("codex")) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::session_pk.ne(naming_session)) - .filter( - sql::>("json_extract(llm_usage_events.metadata, '$.codex_turn_id')") - .between(low, high), - ) - .select((llm_usage_events::session_pk, llm_usage_events::metadata)) - .load::<(String, String)>(connection)?; - let sessions = candidates - .into_iter() - .filter(|(_, metadata)| { - serde_json::from_str::(metadata) - .is_ok_and(|metadata| opens_with_prompt(&metadata, &prompt)) - }) - .map(|(session, _)| session) - .collect::>(); - let mut sessions = sessions.into_iter(); - let task_session = match (sessions.next(), sessions.next()) { - (Some(session), None) => Some(session), - _ => None, - }; - if let Some(parent) = old_parent.as_deref() - && Some(parent) != task_session.as_deref() - { - unmatch( - connection, - naming_session, - parent, - Some(title_event_pk.clone()), - )?; - } - let Some(task_session) = task_session else { - return Ok(()); - }; - diesel::update(session_titles::table.find(naming_session)) - .set(session_titles::parent_session_pk.eq(&task_session)) .execute(connection)?; - put_title( - connection, - TitleUpdate { - session_pk: &task_session, - user_id: &user_id, - title: Some((title, title_event_pk, title_observed_at)), - purpose: None, - }, - ) + Ok(()) } -/// Runs the one-time scans that bring events stored before capture existed up -/// to date. +/// Runs the one-time scan that names sessions stored before titles existed. pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { - run_once(connection, BACKFILL_NAME, backfill_titles)?; - run_once(connection, LINK_BACKFILL_NAME, backfill_links)?; - run_once(connection, LINK_REPAIR_NAME, backfill_links)?; - run_once(connection, DEFAULT_BACKFILL_NAME, backfill_defaults) -} - -fn run_once( - connection: &mut SqliteConnection, - name: &str, - scan: fn(&mut SqliteConnection) -> Result<(), AppError>, -) -> Result<(), AppError> { let done = schema_backfills::table - .find(name) + .find(DEFAULT_BACKFILL_NAME) .select(schema_backfills::name) .first::(connection) .optional()? @@ -469,10 +147,10 @@ fn run_once( return Ok(()); } connection.immediate_transaction(|transaction| { - scan(transaction)?; + backfill_defaults(transaction)?; diesel::insert_into(schema_backfills::table) .values(( - schema_backfills::name.eq(name), + schema_backfills::name.eq(DEFAULT_BACKFILL_NAME), schema_backfills::completed_at.eq(Utc::now().timestamp_micros()), )) .execute(transaction)?; @@ -480,103 +158,12 @@ fn run_once( }) } -/// Captures titles from naming requests stored before capture existed. -fn backfill_titles(connection: &mut SqliteConnection) -> Result<(), AppError> { - let requests = llm_usage_events::table - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::agent_name.eq_any(["codex", "claude-code"])) - .filter( - llm_usage_events::text - .like("%You are naming a coding session%") - .or(llm_usage_events::text.like("%Generate a concise, single-line task title%")), - ) - .order_by(llm_usage_events::observed_at.asc()) - .select(( - llm_usage_events::id, - llm_usage_events::user_id, - llm_usage_events::session_pk, - llm_usage_events::turn_index, - llm_usage_events::agent_name, - llm_usage_events::text, - llm_usage_events::metadata, - llm_usage_events::observed_at, - )) - .load::<( - String, - String, - String, - i32, - String, - Option, - String, - i64, - )>(connection)?; - for (event_pk, user_id, session_pk, turn_index, agent_name, text, metadata, observed_at) in - requests - { - let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); - capture( - connection, - &CapturedEvent { - user_id: parse_uuid(&user_id, "event user id")?, - session_pk: parse_uuid(&session_pk, "event session id")?, - event_pk: parse_uuid(&event_pk, "event id")?, - turn_index, - agent_name: &agent_name, - event_type: "request", - text: text.as_deref(), - metadata: &metadata, - observed_at, - }, - )?; - } - Ok(()) -} - -/// Matches naming threads whose titles were captured before matching existed. -fn backfill_links(connection: &mut SqliteConnection) -> Result<(), AppError> { - let naming_sessions = session_titles::table - .filter(session_titles::purpose.eq(TITLE_GENERATION_PURPOSE)) - .select(session_titles::session_pk) - .load::(connection)?; - for naming_session in naming_sessions { - let request = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(&naming_session)) - .filter(llm_usage_events::event_type.eq("request")) - .filter(llm_usage_events::agent_name.eq("codex")) - .order_by(llm_usage_events::observed_at.desc()) - .select((llm_usage_events::text, llm_usage_events::metadata)) - .first::<(Option, String)>(connection) - .optional()?; - let Some((text, metadata)) = request else { - continue; - }; - let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); - let Some(naming) = NamingRequest::parse("codex", text.as_deref(), &metadata) else { - continue; - }; - diesel::update(session_titles::table.find(&naming_session)) - .set(( - session_titles::naming_turn_id.eq(codex_turn_id(&metadata)), - session_titles::naming_prompt.eq(naming.prompt), - )) - .execute(connection)?; - match_naming_thread(connection, &naming_session)?; - } - Ok(()) -} - /// Visit requests in time order until the first readable user input is found. fn backfill_defaults(connection: &mut SqliteConnection) -> Result<(), AppError> { let sessions = agent_sessions::table - .filter(agent_sessions::agent_name.ne_all(["codex", "claude-code"])) - .select(( - agent_sessions::id, - agent_sessions::user_id, - agent_sessions::agent_name, - )) - .load::<(String, String, String)>(connection)?; - for (session, user_id, agent_name) in sessions { + .select((agent_sessions::id, agent_sessions::user_id)) + .load::<(String, String)>(connection)?; + for (session, user_id) in sessions { let mut offset = 0_i64; loop { let requests = llm_usage_events::table @@ -600,19 +187,20 @@ fn backfill_defaults(connection: &mut SqliteConnection) -> Result<(), AppError> let mut found = false; for (event_pk, observed_at, metadata) in requests { let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); - if default_title(&metadata).is_none() { + let user_input = matches!( + TitleContribution::of("request", &metadata), + Some(TitleContribution::UserInput) + ); + if !user_input || default_title(&metadata).is_none() { continue; } - capture_default( + capture( connection, &CapturedEvent { user_id: parse_uuid(&user_id, "user id")?, session_pk: parse_uuid(&session, "session id")?, event_pk: parse_uuid(&event_pk, "event id")?, - turn_index: 0, - agent_name: &agent_name, event_type: "request", - text: None, metadata: &metadata, observed_at, }, @@ -676,8 +264,6 @@ mod tests { use crate::usage::IngestEventsRequest; const OWNER_ID: Uuid = Uuid::from_u128(1); - const CLAUDE_NAMING: &str = "You are naming a coding session so the user can pick it out"; - const CODEX_NAMING: &str = "Generate a concise, single-line task title of at most 36"; fn database() -> SqliteConnection { let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); @@ -685,20 +271,46 @@ mod tests { database } - fn event(agent: &str, session: &str, id: &str, at: &str, kind: &str, text: &str) -> Value { - json!({ + /// A request whose collector conversation holds one user message. + fn request(agent: &str, session: &str, id: &str, at: &str, input: &str) -> Value { + event( + agent, + session, + id, + at, + "request", + json!({"conversation": { + "messages": [{"role": "user", "text": input}], + "completeness": "complete" + }}), + ) + } + + /// The response of a naming call, carrying the collector's title. + fn titled_response(session: &str, id: &str, at: &str, title: &str) -> Value { + event( + "claude-code", + session, + id, + at, + "response", + json!({"purpose": "session_title", "session_title": title}), + ) + } + + fn event(agent: &str, session: &str, id: &str, at: &str, kind: &str, metadata: Value) -> Value { + let mut event = json!({ "event_id": id, "observed_at": at, "device": {"host_name": "local", "platform": "macos"}, "agent": {"name": agent}, "session_id": session, - "turn_index": 2, "llm": {"provider": "anthropic", "model": "claude-haiku"}, "event_type": kind, - "text": text, - "token_usage": {"output_tokens": 1}, - "metadata": {} - }) + "token_usage": {"output_tokens": 1_i32} + }); + event["metadata"] = metadata; + event } fn ingest(database: &mut SqliteConnection, events: &[Value]) { @@ -721,111 +333,121 @@ mod tests { (title, purpose) } - fn session_pk(database: &mut SqliteConnection, session: &str) -> Uuid { - let pk = agent_sessions::table - .filter(agent_sessions::session_id.eq(session)) - .select(agent_sessions::id) - .first::(database) - .expect("session should exist"); - Uuid::parse_str(&pk).expect("uuid") - } - - fn parent_of(database: &mut SqliteConnection, session: &str) -> Option { - let pk = session_pk(database, session); - load(database, pk).expect("title should load").2 - } - - const NAMING_TURN: &str = "01a0f320-138e-7d21-99ea-03a7ae2a9b50"; - const TASK_TURN: &str = "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"; - const LATE_TURN: &str = "01a0f330-1350-71c2-9c93-5fc5fd7a6bfb"; - const TASK: &str = "画一只鹈鹕骑自行车，不要联网"; - - fn codex_naming_thread(session: &str) -> [Value; 2] { - let mut request = event( - "codex", - session, - &format!("{session}-r"), - "2026-09-30T00:00:01Z", - "request", - &format!("{CODEX_NAMING}\n\nUser prompt:\n画一只鹈鹕骑自行车"), - ); - request["metadata"] = json!({"codex_turn_id": NAMING_TURN}); - let mut response = event( - "codex", - session, - &format!("{session}-s"), - "2026-09-30T00:00:02Z", - "response", - r#"{"title":"Draw a pelican"}"#, - ); - response["metadata"] = json!({"codex_turn_id": NAMING_TURN}); - [request, response] - } - - fn codex_task(session: &str, turn_id: &str, message: &str) -> Value { - let mut request = event( - "codex", - session, - &format!("{session}-{turn_id}"), - "2026-09-30T00:03:00Z", - "request", - message, - ); - request["metadata"] = json!({ - "codex_turn_id": turn_id, - "conversation": {"messages": [ - {"role": "developer", "text": "You are Codex"}, - {"role": "user", "text": "…"}, - {"role": "user", "text": message}, - ]}, - }); - request - } - #[test] - fn names_a_claude_code_session_from_its_own_naming_call() { + fn names_a_session_from_the_collector_title() { let mut database = database(); + let mut naming = request( + "claude-code", + "cc", + "r2", + "2026-09-30T00:00:02Z", + "fix the build", + ); + naming["metadata"]["purpose"] = json!("session_title"); ingest( &mut database, &[ - event( + request( "claude-code", "cc", "r1", "2026-09-30T00:00:00Z", - "request", "fix the build", ), - event( - "claude-code", - "cc", - "s1", - "2026-09-30T00:00:01Z", - "response", - "Done.", - ), - event( - "claude-code", - "cc", - "r2", - "2026-09-30T00:00:02Z", - "request", - CLAUDE_NAMING, - ), - event( + naming, + titled_response("cc", "s2", "2026-09-30T00:00:03Z", "Build fix"), + ], + ); + + assert_eq!( + session_title(&mut database, "cc"), + (Some("Build fix".to_owned()), None) + ); + } + + #[test] + fn agent_titles_outrank_default_titles_in_any_arrival_order() { + for title_first in [false, true] { + let mut database = database(); + let mut events = vec![ + request( "claude-code", "cc", - "s2", - "2026-09-30T00:00:03Z", - "response", - r#"{"title":"Build fix"}"#, + "r1", + "2026-09-30T00:00:00Z", + "fix the build", ), + titled_response("cc", "s1", "2026-09-30T00:00:03Z", "Build fix"), + ]; + // An earlier user request delivered late does not replace the title. + if title_first { + events.reverse(); + } + for value in events { + ingest(&mut database, &[value]); + } + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Build fix"), + "title first: {title_first}" + ); + } + } + + #[test] + fn keeps_the_newest_agent_title() { + let mut database = database(); + ingest( + &mut database, + &[ + titled_response("cc", "s2", "2026-09-30T02:00:01Z", "Newer"), + titled_response("cc", "s1", "2026-09-30T01:00:01Z", "Older"), ], ); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Newer") + ); + ingest( + &mut database, + &[titled_response( + "cc", + "s3", + "2026-09-30T03:00:01Z", + "Newest", + )], + ); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Newest") + ); + } + #[test] + fn title_like_replies_without_the_collector_field_are_ordinary_output() { + let mut database = database(); + let mut reply = event( + "claude-code", + "cc", + "s1", + "2026-09-30T00:00:01Z", + "response", + json!({"conversation": { + "messages": [{"role": "assistant", "text": "{\"title\":\"Quoted\"}"}], + "completeness": "complete" + }}), + ); + reply["text"] = json!("{\"title\":\"Quoted\"}"); + ingest( + &mut database, + &[ + request("claude-code", "cc", "r1", "2026-09-30T00:00:00Z", "Name it"), + reply, + ], + ); assert_eq!( - session_title(&mut database, "cc"), - (Some("Build fix".to_owned()), None) + session_title(&mut database, "cc").0.as_deref(), + Some("Name it") ); } @@ -844,10 +466,8 @@ mod tests { "First", ), ] { - let mut value = event("cursor", "default", id, at, kind, input); - value["metadata"] = json!({"conversation": {"messages": [ - {"role": "user", "text": input} - ]}}); + let mut value = request("cursor", "default", id, at, input); + value["event_type"] = json!(kind); ingest(&mut database, &[value]); assert_eq!( session_title(&mut database, "default").0.as_deref(), @@ -856,6 +476,51 @@ mod tests { } } + #[test] + fn codex_task_sessions_keep_their_first_input_and_naming_threads_are_marked() { + let mut database = database(); + let mut naming = request( + "codex", + "title-thread", + "n1", + "2026-09-30T00:00:01Z", + "Generate a concise, single-line task title.\nUser prompt:\n画一只鹈鹕", + ); + naming["metadata"]["conversation"]["thread_source"] = json!("thread_title"); + let mut task = request( + "codex", + "task", + "t1", + "2026-09-30T00:00:00Z", + "…", + ); + task["metadata"]["conversation"]["thread_source"] = json!("user"); + ingest( + &mut database, + &[ + task, + naming, + event( + "codex", + "title-thread", + "n2", + "2026-09-30T00:00:02Z", + "response", + json!({}), + ), + ], + ); + + assert_eq!( + session_title(&mut database, "task").0.as_deref(), + Some("…") + ); + assert_eq!( + session_title(&mut database, "title-thread"), + (None, Some("title_generation".to_owned())) + ); + } + #[test] fn backfills_default_titles_past_empty_requests_once() { let mut database = database(); @@ -867,20 +532,17 @@ mod tests { &format!("empty-{index}"), "2026-09-30T00:00:00Z", "request", - "No structured user input", + json!({}), ) }) .collect(); - let mut first = event( + values.push(request( "cursor", "default", "first", "2026-09-30T00:00:01Z", - "request", "First request", - ); - archive(&mut first, "user"); - values.push(first); + )); ingest(&mut database, &values); diesel::delete(session_titles::table) .execute(&mut database) @@ -896,356 +558,12 @@ mod tests { .expect("clear titles"); backfill(&mut database).expect("second backfill"); assert_eq!(session_title(&mut database, "default"), (None, None)); - } - - #[test] - fn marks_codex_naming_threads_as_auxiliary_even_when_the_response_arrives_first() { - let mut database = database(); - ingest( - &mut database, - &[event( - "codex", - "title-thread", - "s1", - "2026-09-30T00:00:01Z", - "response", - "{\"title\":\"Draw a pelican\"}\n\n{\"title\":\"Draw a pelican\"}", - )], - ); - ingest( - &mut database, - &[event( - "codex", - "title-thread", - "r1", - "2026-09-30T00:00:00Z", - "request", - CODEX_NAMING, - )], - ); - - assert_eq!( - session_title(&mut database, "title-thread"), - ( - Some("Draw a pelican".to_owned()), - Some("title_generation".to_owned()) - ) - ); - } - - #[test] - fn keeps_the_newest_title_and_ignores_quoted_instructions() { - let mut database = database(); - ingest( - &mut database, - &[ - event( - "claude-code", - "cc", - "r2", - "2026-09-30T02:00:00Z", - "request", - CLAUDE_NAMING, - ), - event( - "claude-code", - "cc", - "s2", - "2026-09-30T02:00:01Z", - "response", - r#"{"title":"Newer"}"#, - ), - event( - "claude-code", - "cc", - "r1", - "2026-09-30T01:00:00Z", - "request", - CLAUDE_NAMING, - ), - event( - "claude-code", - "cc", - "s1", - "2026-09-30T01:00:01Z", - "response", - r#"{"title":"Older"}"#, - ), - event( - "claude-code", - "cc", - "r3", - "2026-09-30T03:00:00Z", - "request", - CLAUDE_NAMING, - ), - event( - "claude-code", - "cc", - "s3", - "2026-09-30T03:00:01Z", - "response", - "That prompt names sessions.", - ), - ], - ); - - assert_eq!( - session_title(&mut database, "cc").0.as_deref(), - Some("Newer") - ); - } - - #[test] - fn backfills_stored_naming_calls_once() { - let mut database = database(); - ingest( - &mut database, - &[ - event( - "claude-code", - "cc", - "r1", - "2026-09-30T00:00:00Z", - "request", - CLAUDE_NAMING, - ), - event( - "claude-code", - "cc", - "s1", - "2026-09-30T00:00:01Z", - "response", - r#"{"title":"Stored"}"#, - ), - ], - ); - diesel::delete(session_titles::table) - .execute(&mut database) - .expect("clear titles"); - - backfill(&mut database).expect("backfill should succeed"); - assert_eq!( - session_title(&mut database, "cc").0.as_deref(), - Some("Stored") - ); - - diesel::delete(session_titles::table) - .execute(&mut database) - .expect("clear titles"); - backfill(&mut database).expect("second backfill should succeed"); - assert_eq!(session_title(&mut database, "cc"), (None, None)); assert_eq!( schema_backfills::table .count() .get_result::(&mut database) .expect("count"), - 4, - "generated titles, both link versions and default titles are each scanned once" - ); - } - - #[test] - fn names_the_codex_task_its_naming_thread_quotes_in_either_order() { - for naming_first in [true, false] { - let mut database = database(); - let naming = codex_naming_thread("title-thread"); - let task = codex_task("task", TASK_TURN, TASK); - if naming_first { - ingest(&mut database, &naming); - ingest(&mut database, &[task]); - } else { - ingest(&mut database, &[task]); - ingest(&mut database, &naming); - } - - assert_eq!( - session_title(&mut database, "task"), - (Some("Draw a pelican".to_owned()), None) - ); - let task_pk = session_pk(&mut database, "task"); - assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); - } - } - - #[test] - fn leaves_codex_naming_threads_unmatched_without_one_clear_task() { - let mut database = database(); - ingest(&mut database, &codex_naming_thread("title-thread")); - ingest( - &mut database, - &[ - codex_task("other-message", TASK_TURN, "写一个排序函数"), - codex_task("too-late", LATE_TURN, TASK), - ], - ); - assert_eq!(parent_of(&mut database, "title-thread"), None); - assert_eq!(session_title(&mut database, "too-late"), (None, None)); - - // The same message sent twice at once cannot be told apart. - ingest( - &mut database, - &[ - codex_task("first", TASK_TURN, TASK), - codex_task("second", TASK_TURN, TASK), - ], - ); - assert_eq!(parent_of(&mut database, "title-thread"), None); - assert_eq!(session_title(&mut database, "second"), (None, None)); - } - fn archive(event: &mut Value, source: &str) { - use base64::{Engine as _, engine::general_purpose::STANDARD}; - let turn = event["metadata"]["codex_turn_id"].clone(); - let payload = json!({ - "input": [ - {"role": "developer", "content": "Instructions"}, - {"role": "user", "content": [{"type": "input_text", "text": event["text"]}]} - ], - "client_metadata": {"x-codex-turn-metadata": json!({ - "thread_source": source - }).to_string()} - }); - event["metadata"] = json!({"codex_turn_id": turn, "raw_body": {"parts": [{ - "truncated": false, "content_base64": STANDARD.encode(payload.to_string()) - }]}}); - } - - #[test] - fn raw_codex_links_work_for_every_arrival_order() { - for naming_first in [false, true] { - for response_first in [false, true] { - let mut database = database(); - let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title"); - // Raw-only requests also work when the flattened text is absent. - naming[0]["text"] = Value::Null; - if response_first { - naming.reverse(); - } - let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user"); - if naming_first { - ingest(&mut database, &naming); - ingest(&mut database, &[task]); - } else { - ingest(&mut database, &[task]); - ingest(&mut database, &naming); - } - assert_eq!( - session_title(&mut database, "task").0.as_deref(), - Some("Draw a pelican") - ); - let task_pk = session_pk(&mut database, "task"); - assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); - } - } - } - - #[test] - fn other_devices_and_auxiliary_threads_do_not_withdraw_links() { - let mut database = database(); - let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title"); - let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user"); - ingest(&mut database, &[task]); - ingest(&mut database, &naming); - let task_pk = session_pk(&mut database, "task"); - for (session, host, source) in [ - ("other-device", "another-host", "user"), - ("guardian", "local", "guardian_review"), - ("subagent", "local", "subagent"), - ] { - let mut request = codex_task(session, TASK_TURN, TASK); - request["device"]["host_name"] = json!(host); - archive(&mut request, source); - ingest(&mut database, &[request]); - assert_eq!( - parent_of(&mut database, "title-thread"), - Some(task_pk), - "{session}" - ); - assert_eq!( - session_title(&mut database, "task").0.as_deref(), - Some("Draw a pelican") - ); - } - let mut conflict = codex_task("second-task", TASK_TURN, TASK); - archive(&mut conflict, "user"); - ingest(&mut database, &[conflict]); - assert_eq!(parent_of(&mut database, "title-thread"), None); - assert_eq!(session_title(&mut database, "task").0, None); - } - - #[test] - fn repairs_existing_unlinked_raw_titles_once() { - let mut database = database(); - let mut naming = codex_naming_thread("title-thread"); - archive(&mut naming[0], "thread_title"); - let mut task = codex_task("task", TASK_TURN, TASK); - archive(&mut task, "user"); - ingest(&mut database, &[task]); - ingest(&mut database, &naming); - let task_pk = session_pk(&mut database, "task"); - let naming_pk = session_pk(&mut database, "title-thread"); - diesel::delete(session_titles::table.find(task_pk.to_string())) - .execute(&mut database) - .expect("remove projection"); - diesel::update(session_titles::table.find(naming_pk.to_string())) - .set(session_titles::parent_session_pk.eq(None::)) - .execute(&mut database) - .expect("simulate old raw-body mismatch"); - for name in [super::BACKFILL_NAME, super::LINK_BACKFILL_NAME] { - diesel::insert_into(schema_backfills::table) - .values(( - schema_backfills::name.eq(name), - schema_backfills::completed_at.eq(0_i64), - )) - .execute(&mut database) - .expect("mark older scans completed"); - } - backfill(&mut database).expect("repair"); - assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); - assert_eq!( - session_title(&mut database, "task").0.as_deref(), - Some("Draw a pelican") + 1 ); - backfill(&mut database).expect("idempotent repair"); - assert_eq!(parent_of(&mut database, "title-thread"), Some(task_pk)); - } - - #[test] - fn generic_agents_are_not_classified_by_codex_or_claude_templates() { - let mut database = database(); - for (session, instruction) in [ - ("generic-cc", CLAUDE_NAMING), - ("generic-codex", CODEX_NAMING), - ] { - ingest( - &mut database, - &[ - event( - "cursor", - session, - &format!("{session}-r"), - "2026-09-30T00:00:01Z", - "request", - instruction, - ), - event( - "cursor", - session, - &format!("{session}-s"), - "2026-09-30T00:00:02Z", - "response", - r#"{"title":"Quoted title"}"#, - ), - ], - ); - assert_eq!(session_title(&mut database, session), (None, None)); - } - backfill(&mut database).expect("historical scan follows the same adapter policy"); - assert_eq!(session_title(&mut database, "generic-cc"), (None, None)); - assert_eq!(session_title(&mut database, "generic-codex"), (None, None)); } } diff --git a/crates/abyss-backend/src/usage/conversation.rs b/crates/abyss-backend/src/usage/conversation.rs new file mode 100644 index 0000000..9b9a5a9 --- /dev/null +++ b/crates/abyss-backend/src/usage/conversation.rs @@ -0,0 +1,111 @@ +//! Typed access to collector-produced conversation projections; no provider decoding. + +use serde::Deserialize; +use serde_json::Value; + +#[derive(Deserialize)] +pub struct Conversation { + pub messages: Vec, + #[serde(default)] + pub previous_response_id: Option, + #[serde(default)] + pub instructions_present: bool, + #[serde(default)] + completeness: Option, +} + +#[derive(Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +enum Completeness { + Complete, + Partial, + Unsupported, +} + +/// Message roles the collector emits. Roles outside this set, including the +/// placeholder `request` from older collectors, read as [`Role::Unknown`]. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum Role { + System, + Developer, + User, + Assistant, + Reasoning, + Tool, + #[serde(other)] + Unknown, +} + +impl Role { + pub const fn as_str(self) -> &'static str { + match self { + Self::System => "system", + Self::Developer => "developer", + Self::User => "user", + Self::Assistant => "assistant", + Self::Reasoning => "reasoning", + Self::Tool => "tool", + Self::Unknown => "unknown", + } + } +} + +#[derive(Clone, Deserialize)] +pub struct Message { + #[serde(default)] + pub id: Option, + pub role: Role, + pub text: String, +} + +impl Conversation { + /// Whether the event carries a conversation from a collector that reports + /// completeness. Older collectors stored messages without it; their + /// requests keep using the flattened text where they did before. + pub fn declared(metadata: &Value) -> bool { + metadata.pointer("/conversation/completeness").is_some() + } + + /// Thread source the Agent declared for the request, read without + /// decoding its messages. + pub fn thread_source(metadata: &Value) -> Option<&str> { + metadata.get("conversation")?.get("thread_source")?.as_str() + } + + pub fn read(metadata: &Value) -> Option { + let value = metadata.get("conversation")?; + let conversation: Self = Self::deserialize(value).ok()?; + if conversation + .completeness + .as_ref() + .is_some_and(|state| *state != Completeness::Complete) + { + return None; + } + Some(conversation) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_conversations_read_with_unknown_roles() { + let metadata = serde_json::json!({"conversation": {"messages": [ + {"role": "request", "text": "placeholder"}, + {"role": "function", "text": "legacy"}, + {"role": "user", "text": "hello"} + ]}}); + let conversation = Conversation::read(&metadata).expect("legacy shape"); + assert_eq!( + conversation + .messages + .iter() + .map(|message| message.role) + .collect::>(), + [Role::Unknown, Role::Unknown, Role::User] + ); + } +} diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 54d813c..035f9b8 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -7,11 +7,11 @@ /// Image attachment contracts and validation. pub mod attachments; +mod conversation; /// Opaque diagnostic-capture ingest contracts. pub mod diagnostics; pub mod event_order; pub mod persistence; -mod provider_body; /// Owner-scoped session index contracts. pub mod session_list; pub mod session_title; @@ -47,10 +47,6 @@ pub struct IngestUsageEvent { pub agent: AgentPayload, /// Agent-native session identifier. pub session_id: String, - /// Collector-side best-effort turn number. Ingest normalizes this when a - /// stable logical-turn/provider identity shows that a restarted collector - /// reused an earlier local sequence number. - pub turn_index: i32, /// LLM provider and model labels. pub llm: LlmPayload, /// Whether the event represents the request or response side of a call. diff --git a/crates/abyss-backend/src/usage/persistence.rs b/crates/abyss-backend/src/usage/persistence.rs index 0e32d6d..cdfd0fc 100644 --- a/crates/abyss-backend/src/usage/persistence.rs +++ b/crates/abyss-backend/src/usage/persistence.rs @@ -60,9 +60,6 @@ pub fn validate_event(event: &IngestUsageEvent) -> Result<(), AppError> { require_non_empty(&event.session_id, "session_id")?; require_non_empty(&event.llm.provider, "llm.provider")?; require_non_empty(&event.llm.model, "llm.model")?; - if event.turn_index < 1_i32 { - return Err(AppError::validation("turn_index must be >= 1".to_owned())); - } let tokens = normalized_tokens(event)?; if tokens.total == 0 { return Ok(()); @@ -304,6 +301,7 @@ pub struct TurnIdentity { pub enum TurnIdentityKind { CodexTurnId, ClaudeTurnId, + TurnId, ResponseId, MessageId, RequestHash, @@ -314,6 +312,7 @@ impl TurnIdentityKind { match self { Self::CodexTurnId => "codex_turn_id", Self::ClaudeTurnId => "claude_turn_id", + Self::TurnId => "turn_id", Self::ResponseId => "response_id", Self::MessageId => "message_id", Self::RequestHash => "request_hash", @@ -331,6 +330,7 @@ pub fn turn_identity_from_metadata(metadata: &serde_json::Value) -> Option Option, +} + +/// Billing-header field naming the user prompt a Claude Code request serves. +const CLAUDE_CODE_PROMPT_ID_KEY: &str = "cc_prompt_id"; +const CLAUDE_CODE_BILLING_KEY: &str = "claude_code_billing"; + +/// Storage resolves an attachment in order: the turn of a main-loop event with +/// the same prompt id, the latest turn started at or before the event, the +/// earliest turn, and only for a session without turns a turn of its own. +/// Other Agents and Claude Code main-loop events keep their turn identity. +pub fn session_turn_attachment( + agent_name: &str, + metadata: &serde_json::Value, +) -> Option { + if agent_name != "claude-code" + || metadata_string(metadata, TurnIdentityKind::ClaudeTurnId.metadata_key()).is_some() + { + return None; + } + Some(SessionTurnAttachment { + prompt_id: metadata + .get(CLAUDE_CODE_BILLING_KEY) + .and_then(|billing| metadata_string(billing, CLAUDE_CODE_PROMPT_ID_KEY)), + }) +} + fn metadata_string(metadata: &serde_json::Value, key: &str) -> Option { metadata .get(key) @@ -350,28 +383,13 @@ fn metadata_string(metadata: &serde_json::Value, key: &str) -> Option { .map(str::to_owned) } -pub const fn choose_turn_index( - requested_turn_index: i32, - existing_turn_index: Option, - next_turn_index: i32, -) -> i32 { - if let Some(existing_turn_index) = existing_turn_index { - return existing_turn_index; - } - if requested_turn_index < next_turn_index { - next_turn_index - } else { - requested_turn_index - } -} - #[cfg(test)] mod tests { use serde_json::json; use super::{ - TurnIdentityKind, canonical_agent_name, choose_turn_index, escape_like_pattern, - parse_group_by, turn_identity_from_metadata, + TurnIdentityKind, canonical_agent_name, escape_like_pattern, parse_group_by, + session_turn_attachment, turn_identity_from_metadata, }; #[test] @@ -382,7 +400,7 @@ mod tests { } #[test] - fn turn_identity_and_restart_resolution_are_stable() { + fn agent_turn_identity_wins_over_provider_identities() { let identity = turn_identity_from_metadata(&json!({ "response_id": "response", "codex_turn_id": "turn" @@ -390,7 +408,30 @@ mod tests { .expect("turn identity should be extracted"); assert!(matches!(identity.kind, TurnIdentityKind::CodexTurnId)); assert_eq!(identity.value, "turn"); - assert_eq!(choose_turn_index(1_i32, None, 4_i32), 4_i32); - assert_eq!(choose_turn_index(1_i32, Some(2_i32), 4_i32), 2_i32); + let identity = turn_identity_from_metadata(&json!({ + "response_id": "response", + "turn_id": "resp_1" + })) + .expect("turn identity should be extracted"); + assert!(matches!(identity.kind, TurnIdentityKind::TurnId)); + } + + #[test] + fn only_claude_code_events_without_turn_id_attach_to_a_session_turn() { + let billing = json!({"claude_code_billing": {"cc_prompt_id": " prompt "}}); + let attachment = session_turn_attachment("claude-code", &billing) + .expect("a Claude Code side request should attach"); + assert_eq!(attachment.prompt_id.as_deref(), Some("prompt")); + let side = session_turn_attachment("claude-code", &json!({"message_id": "msg"})) + .expect("a request without billing header should attach"); + assert_eq!(side.prompt_id, None); + assert!( + session_turn_attachment( + "claude-code", + &json!({"claude_turn_id": "cc_turn:3", "claude_code_billing": {"cc_prompt_id": "p"}}) + ) + .is_none() + ); + assert!(session_turn_attachment("codex", &billing).is_none()); } } diff --git a/crates/abyss-backend/src/usage/provider_body.rs b/crates/abyss-backend/src/usage/provider_body.rs deleted file mode 100644 index 4fb9732..0000000 --- a/crates/abyss-backend/src/usage/provider_body.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! Stateless decoding of archived provider content, shared by projections. - -use base64::{Engine as _, engine::general_purpose::STANDARD}; -use serde_json::Value; - -#[derive(Clone)] -pub struct Message { - pub id: Option, - pub role: String, - pub text: String, -} - -pub fn request_payload(values: &[Value]) -> Option<&Value> { - values - .iter() - .map(|v| v.get("response").unwrap_or(v)) - .find(|v| v.get("input").is_some() || v.get("messages").is_some()) -} - -/// Decode bytes only for projection. The original base64, whitespace, unknown -/// fields, and SSE events remain untouched in event metadata. -pub fn bodies(metadata: &Value) -> Option> { - let parts = metadata.get("raw_body")?.get("parts")?.as_array()?; - let mut values = Vec::new(); - for part in parts { - if part.get("truncated").and_then(Value::as_bool) != Some(false) - || part - .get("content_encoding") - .and_then(Value::as_str) - .is_some_and(|v| v != "identity") - { - return None; - } - let bytes = STANDARD - .decode(part.get("content_base64")?.as_str()?) - .ok()?; - if let Ok(value) = serde_json::from_slice(&bytes) { - values.push(value); - } else { - let text = std::str::from_utf8(&bytes).ok()?; - let mut data = String::new(); - for line in text.lines().chain(std::iter::once("")) { - if line.is_empty() { - if let Ok(value) = serde_json::from_str(&data) { - values.push(value); - } - data.clear(); - } else if let Some(line) = line.strip_prefix("data:") { - if !data.is_empty() { - data.push('\n'); - } - data.push_str(line.strip_prefix(' ').unwrap_or(line)); - } - } - } - } - Some(values) -} - -pub fn request_messages(payload: &Value) -> Vec { - let mut result = Vec::new(); - for (key, role) in [("instructions", "developer"), ("system", "system")] { - if let Some(value) = payload.get(key) { - result.push(Message { - id: None, - role: role.to_owned(), - text: text(value), - }); - } - } - if let Some(input) = payload.get("input").or_else(|| payload.get("messages")) { - if let Some(items) = input.as_array() { - result.extend(items.iter().filter_map(|v| message(v, "request"))); - } else { - result.extend(message(input, "request")); - } - } - result -} - -pub fn message(value: &Value, default_role: &str) -> Option { - let kind = value - .get("type") - .and_then(Value::as_str) - .unwrap_or_default(); - let role = value - .get("role") - .and_then(Value::as_str) - .unwrap_or(default_role); - if role == "tool" || tool(kind) { - return None; - } - let (role, content) = if kind == "reasoning" { - ("reasoning", value.get("summary")?) - } else { - (role, value.get("content").unwrap_or(value)) - }; - Some(Message { - id: value.get("id").and_then(Value::as_str).map(str::to_owned), - role: role.to_owned(), - text: text(content), - }) -} - -pub fn text(value: &Value) -> String { - match value { - Value::String(value) => value.clone(), - Value::Array(values) => values - .iter() - .map(text) - .filter(|v| !v.is_empty()) - .collect::>() - .join("\n"), - Value::Object(obj) => { - let kind = obj.get("type").and_then(Value::as_str).unwrap_or_default(); - if tool(kind) { - return String::new(); - } - obj.get("text") - .or_else(|| obj.get("content")) - .or_else(|| obj.get("thinking")) - .or_else(|| obj.get("output_text")) - .map(text) - .unwrap_or_default() - } - _ => String::new(), - } -} - -fn tool(kind: &str) -> bool { - matches!(kind, "tool_use" | "tool_result" | "tool_call") - || kind.ends_with("_call") - || kind.ends_with("_call_output") -} diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index ae02a1c..6cf38e5 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -1,176 +1,64 @@ -//! Agent-generated session names, or first-user-input previews for other Agents. +//! Session names: the title an Agent generated for its own session, else the +//! session's first user input. //! -//! Claude Code and Codex ask a model to name each session and record the -//! answer as `{"title": "..."}`. Those calls are captured like any other, so -//! the name can be read back from them without guessing: a call counts only -//! when its request carries an official naming instruction and its response -//! consists solely of title objects that agree. +//! The collector reports a naming call as event fields: both sides carry a +//! `purpose`, and the response carries the `session_title` it produced for its +//! own session. Titles are therefore read without interpreting message text. //! -//! Codex asks from a separate thread whose requests carry no link to the task -//! they name. The naming request quotes the task's user message, and Codex -//! starts the naming turn right after the task turn, so a task turn is matched -//! when it starts close to the naming turn and its user message opens with the -//! quoted prompt. +//! Codex names a task from a separate thread whose requests do not say which +//! session they name, so its task sessions keep their first user input and the +//! naming threads are only marked, from the protocol's `thread_source`. -use serde::Deserialize; use serde_json::Value; -use uuid::Uuid; -use super::provider_body::{bodies, request_messages, request_payload}; +use super::conversation::{Conversation, Role}; -/// Claude Code instruction for naming the session it runs in. -const CLAUDE_CODE_NAMING_INSTRUCTION: &str = "You are naming a coding session"; -/// Codex instruction for naming a task; Codex sends it from its own thread. -const CODEX_NAMING_INSTRUCTION: &str = "Generate a concise, single-line task title"; -/// Longest title kept, in characters; longer answers are not titles. -const MAX_TITLE_CHARS: usize = 200; - -/// Precedes the quoted task message in a Codex naming request. -const CODEX_PROMPT_MARKER: &str = "User prompt:\n"; -/// Widest start gap between a Codex naming turn and the task turn it names. -/// Observed gaps are mostly under half a second and at most about 31 s. -const CODEX_LINK_WINDOW_MS: u64 = 60_000; - -/// Session purpose stored for Codex's standalone naming threads. +/// Session purpose stored for threads that name another session. pub const TITLE_GENERATION_PURPOSE: &str = "title_generation"; - -/// Where a naming call's title belongs. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum TitleTask { - /// The call names the session it was sent from (Claude Code). - NamesOwnSession, - /// The call runs in a separate naming session (Codex); that session is - /// auxiliary and can be linked to the task session it names. - AuxiliarySession, -} - -impl TitleTask { - /// Recognizes a naming call from its request text. - pub fn from_request(agent_name: &str, text: &str) -> Option { - if agent_name == "claude-code" && text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) { - Some(Self::NamesOwnSession) - } else if agent_name == "codex" && text.contains(CODEX_NAMING_INSTRUCTION) { - Some(Self::AuxiliarySession) - } else { - None - } - } -} - -/// Agent-specific interpretation of a naming request. Other agents use the -/// ordinary session projection, even when they quote a naming instruction. -pub struct NamingRequest { - /// Whether the title belongs to this session or a separate task. - pub task: TitleTask, - /// User prompt quoted by Codex's naming instruction. - pub prompt: Option, +/// Thread source Codex declares on its naming threads. +const TITLE_THREAD_SOURCE: &str = "thread_title"; +/// Longest default title, in characters. +const DEFAULT_TITLE_CHARS: usize = 80; + +/// What one stored event contributes to its session's name. +#[derive(Debug, PartialEq, Eq)] +pub enum TitleContribution<'a> { + /// Title the Agent generated for this event's session. + Agent(&'a str), + /// An ordinary request; its first user input may name the session. + UserInput, + /// The session is a thread that names another session. + TitleThread, } -impl NamingRequest { - /// Reads protocol roles from complete bodies, with legacy text compatibility. - pub fn parse(agent_name: &str, text: Option<&str>, metadata: &Value) -> Option { - if !matches!(agent_name, "codex" | "claude-code") { - return None; +impl<'a> TitleContribution<'a> { + /// Classifies an event from collector fields without reading its messages. + /// Requests the Agent made for its own purposes, such as naming calls, are + /// not user input. + pub fn of(event_type: &str, metadata: &'a Value) -> Option { + if let Some(title) = metadata.get("session_title").and_then(Value::as_str) { + return Some(Self::Agent(title)); } - // Most requests are ordinary work. Avoid decoding their archives just - // to discover that they do not contain a naming instruction. - if text.is_some_and(|text| TitleTask::from_request(agent_name, text).is_none()) { + if event_type != "request" || metadata.get("purpose").is_some() { return None; } - if agent_name == "claude-code" { - let instruction = if metadata.get("raw_body").is_some() { - let values = bodies(metadata)?; - request_messages(request_payload(&values)?) - .into_iter() - .find(|message| { - matches!(message.role.as_str(), "system" | "developer" | "user") - && message.text.contains(CLAUDE_CODE_NAMING_INSTRUCTION) - })? - .text - } else { - text?.to_owned() - }; - return Some(Self { - task: TitleTask::from_request(agent_name, &instruction)?, - prompt: None, - }); - } - let text = if metadata.get("raw_body").is_some() { - codex_user_message(metadata, "thread_title")? - } else { - text?.to_owned() - }; - Some(Self { - task: TitleTask::from_request(agent_name, &text)?, - prompt: codex_naming_prompt(&text).map(str::to_owned), - }) - } -} - -/// Reads the last user message from a Codex request. The protocol source, when -/// present, distinguishes title tasks from ordinary work and auxiliary calls. -fn codex_user_message(metadata: &Value, source: &str) -> Option { - if metadata.get("raw_body").is_some() { - let values = bodies(metadata)?; - let payload = request_payload(&values)?; - let client = &payload["client_metadata"]; - let context = match client.get("x-codex-turn-metadata") { - Some(encoded) => serde_json::from_str::(encoded.as_str()?).ok()?, - None => client.clone(), - }; - if context["thread_source"] - .as_str() - .is_some_and(|value| value != source) - { - return None; + if Conversation::thread_source(metadata) == Some(TITLE_THREAD_SOURCE) { + return Some(Self::TitleThread); } - return request_messages(payload) - .into_iter() - .rev() - .find(|message| message.role == "user") - .map(|message| message.text); + Some(Self::UserInput) } - metadata - .pointer("/conversation/messages")? - .as_array()? - .iter() - .rev() - .find(|message| message["role"] == "user")?["text"] - .as_str() - .map(str::to_owned) -} - -/// A naming prompt may be a truncated prefix of the task's user message. -pub fn opens_with_prompt(metadata: &Value, prompt: &str) -> bool { - codex_user_message(metadata, "user").is_some_and(|text| text.trim_start().starts_with(prompt)) } /// Default name: the first non-empty user input, on one line, at most 80 chars. pub fn default_title(metadata: &Value) -> Option { - let input = if metadata.get("raw_body").is_some() { - let values = bodies(metadata)?; - let payload = request_payload(&values)?; - if let Some(input) = payload.get("input").and_then(Value::as_str) { - input.to_owned() - } else { - request_messages(payload) - .into_iter() - .find(|message| message.role == "user" && !message.text.trim().is_empty())? - .text - } - } else { - metadata - .pointer("/conversation/messages")? - .as_array()? - .iter() - .filter(|message| message["role"] == "user") - .filter_map(|message| message["text"].as_str()) - .find(|text| !text.trim().is_empty())? - .to_owned() - }; + let input = Conversation::read(metadata)? + .messages + .into_iter() + .find(|message| message.role == Role::User && !message.text.trim().is_empty())? + .text; let input = input.split_whitespace().collect::>().join(" "); let mut chars = input.chars(); - let mut title: String = chars.by_ref().take(80).collect(); + let mut title: String = chars.by_ref().take(DEFAULT_TITLE_CHARS).collect(); if chars.next().is_some() { title.pop(); title.push('…'); @@ -178,201 +66,41 @@ pub fn default_title(metadata: &Value) -> Option { (!title.is_empty()).then_some(title) } -#[derive(Deserialize)] -struct TitleObject { - title: String, -} - -/// Reads the title from a naming response. -/// -/// Streaming clients may repeat the answer, so several identical objects are -/// accepted. Anything else, such as reasoning text or a normal reply to a -/// user who quoted the naming instruction, yields no title. -pub fn title_from_response(text: &str) -> Option { - let mut title: Option = None; - for value in serde_json::Deserializer::from_str(text.trim()).into_iter::() { - let candidate = value.ok()?.title.trim().to_owned(); - if candidate.is_empty() || candidate.chars().count() > MAX_TITLE_CHARS { - return None; - } - match &title { - Some(existing) if *existing != candidate => return None, - Some(_) => {} - None => title = Some(candidate), - } - } - title -} - -/// Returns `true` when a response text could carry a title, so ingestion can -/// skip the pairing lookup for ordinary replies. -pub fn may_carry_title(text: &str) -> bool { - let trimmed = text.trim_start(); - trimmed.starts_with('{') && trimmed.contains("\"title\"") -} - -/// Returns the task message a Codex naming request quotes. Codex may cut long -/// messages short, so the quote can be a prefix of the task message. -pub fn codex_naming_prompt(request_text: &str) -> Option<&str> { - let (_, prompt) = request_text.split_once(CODEX_PROMPT_MARKER)?; - Some(prompt.trim()).filter(|prompt| !prompt.is_empty()) -} - -/// Returns the Codex turn ID an event's collector recorded. -pub fn codex_turn_id(metadata: &Value) -> Option<&str> { - metadata.get("codex_turn_id")?.as_str() -} - -/// Returns the range of Codex turn IDs that start within the link window of -/// `turn_id`. Codex turn IDs are version 7 UUIDs, whose text sorts by start -/// time held in the top 48 bits. -pub fn codex_turn_id_range(turn_id: &str) -> Option<(String, String)> { - const TIME_SHIFT: u32 = 80; - let uuid = Uuid::parse_str(turn_id).ok()?; - if uuid.get_version_num() != 7 { - return None; - } - let started_ms = u64::try_from(uuid.as_u128().checked_shr(TIME_SHIFT)?).ok()?; - let low = - u128::from(started_ms.saturating_sub(CODEX_LINK_WINDOW_MS)).checked_shl(TIME_SHIFT)?; - let high = u128::from(started_ms.saturating_add(CODEX_LINK_WINDOW_MS)) - .checked_shl(TIME_SHIFT)? - | u128::MAX.checked_shr(u128::BITS - TIME_SHIFT)?; - Some(( - Uuid::from_u128(low).to_string(), - Uuid::from_u128(high).to_string(), - )) -} - #[cfg(test)] mod tests { - use super::*; - - #[test] - fn recognizes_official_naming_instructions() { - assert_eq!( - TitleTask::from_request( - "claude-code", - "system: …\n\nYou are naming a coding session so the user…" - ), - Some(TitleTask::NamesOwnSession) - ); - assert_eq!( - TitleTask::from_request( - "codex", - "Generate a concise, single-line task title of at most 36" - ), - Some(TitleTask::AuxiliarySession) - ); - assert_eq!( - TitleTask::from_request("codex", "please name this session"), - None - ); - } + use serde_json::json; - #[test] - fn reads_single_and_repeated_title_objects() { - assert_eq!( - title_from_response(r#"{"title":"Timeline颗粒度细化"}"#).as_deref(), - Some("Timeline颗粒度细化") - ); - assert_eq!( - title_from_response("{\"title\":\"Send a ping\"}\n\n{\"title\":\"Send a ping\"}") - .as_deref(), - Some("Send a ping") - ); - } + use super::*; - #[test] - fn rejects_everything_that_is_not_one_agreed_title() { - for text in [ - "We need respond title only, imperative verb, under 5 words", - r#"{"title":"A"}{"title":"B"}"#, - r#"{"title":""}"#, - r#"{"name":"x"}"#, - r#"Here is the title: {"title":"x"}"#, - "", - ] { - assert_eq!(title_from_response(text), None, "{text:?}"); + /// Metadata from the current collector, which always reports completeness. + fn request(conversation: &Value) -> Value { + let mut conversation = conversation.clone(); + if conversation.get("completeness").is_none() { + conversation["completeness"] = json!("complete"); } - assert_eq!( - title_from_response(&format!(r#"{{"title":"{}"}}"#, "x".repeat(201))), - None - ); + json!({"conversation": conversation}) } - #[test] - fn reads_the_quoted_task_from_codex_naming_requests() { - assert_eq!( - codex_naming_prompt("Generate a concise title.\n\nUser prompt:\n画一只鹈鹕\n"), - Some("画一只鹈鹕") - ); - assert_eq!(codex_naming_prompt("User prompt:\n "), None); - assert_eq!(codex_naming_prompt("no quote"), None); + fn default_of(messages: &Value) -> Option { + default_title(&request(&json!({"messages": messages}))) } #[test] - fn bounds_codex_turn_ids_by_start_time() { - let (low, high) = - codex_turn_id_range("01a0f320-138e-7d21-99ea-03a7ae2a9b50").expect("UUIDv7"); - assert!(low.as_str() < "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"); - assert!(high.as_str() > "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb"); - assert!(high.as_str() < "01a0f321-0000-7000-8000-000000000000"); - assert_eq!(codex_turn_id_range("turn-1"), None); + fn default_title_reads_the_first_nonempty_user_input() { assert_eq!( - codex_turn_id_range("f81d4fae-7dec-11d0-a765-00a0c91e6bf6"), - None + default_of(&json!([ + {"role":"system","text":"Instructions"}, + {"role":"assistant","text":"Previous reply"}, + {"role":"tool","text":"Tool output"}, + {"role":"user","text":" "}, + {"role":"user","text":" 修复\n 登录问题 "}, + {"role":"user","text":"Second request"} + ])) + .as_deref(), + Some("修复 登录问题") ); } - #[test] - fn matches_the_latest_user_message_by_prefix() { - let metadata = serde_json::json!({"conversation": {"messages": [ - {"role": "user", "text": "…"}, - {"role": "user", "text": "画一只鹈鹕骑自行车，不要联网"}, - {"role": "assistant", "text": "好的"}, - ]}}); - assert!(opens_with_prompt(&metadata, "画一只鹈鹕")); - assert!(!opens_with_prompt(&metadata, "")); - assert!(!opens_with_prompt(&serde_json::json!({}), "画")); - } - - #[test] - fn prefilters_candidate_responses() { - assert!(may_carry_title(" {\"title\": \"x\"}")); - assert!(!may_carry_title("The title is x")); - } - fn archived_request(payload: &Value, turn_id: &str) -> Value { - use base64::{Engine as _, engine::general_purpose::STANDARD}; - serde_json::json!({ - "codex_turn_id": turn_id, - "raw_body": {"parts": [{"truncated": false, "content_base64": - STANDARD.encode(serde_json::to_vec(payload).expect("json"))}]} - }) - } - - #[test] - fn default_title_reads_the_first_nonempty_user_input() { - let messages = serde_json::json!([ - {"role": "system", "content": "System instructions"}, - {"role": "assistant", "content": "Assistant output"}, - {"role": "tool", "content": "Tool output"}, - {"role": "user", "content": " "}, - {"role": "user", "content": [{"type": "text", "text": " 修复\n 登录问题 "}]}, - {"role": "user", "content": "Second request"} - ]); - for key in ["input", "messages"] { - let metadata = archived_request(&serde_json::json!({key: messages}), "unused"); - assert_eq!(default_title(&metadata).as_deref(), Some("修复 登录问题")); - } - let legacy = serde_json::json!({"conversation": {"messages": [ - {"role": "system", "text": "Instructions"}, - {"role": "user", "text": "First request"}, - {"role": "user", "text": "Second request"} - ]}}); - assert_eq!(default_title(&legacy).as_deref(), Some("First request")); - } - #[test] fn default_title_truncates_unicode_and_leaves_short_input_intact() { for (input, expected) in [ @@ -380,102 +108,87 @@ mod tests { ("修".repeat(80), "修".repeat(80)), ("修".repeat(81), format!("{}…", "修".repeat(79))), ] { - let metadata = archived_request(&serde_json::json!({"input": input}), "unused"); - assert_eq!(default_title(&metadata), Some(expected)); + assert_eq!( + default_of(&json!([{"role":"user","text":input}])), + Some(expected) + ); } } #[test] fn default_title_requires_readable_user_input() { - for payload in [ - serde_json::json!({"input": " \n "}), - serde_json::json!({"messages": [{"role": "system", "content": "Instructions"}]}), - serde_json::json!({"messages": [{"role": "user", "content": [ - {"type": "tool_result", "content": "Output"} - ]}]}), + for messages in [ + json!([]), + json!([{"role":"system","text":"Instructions"}]), + json!([{"role":"user","text":" \n "}]), + json!([{"role":"tool","text":"Output"}]), ] { - assert_eq!(default_title(&archived_request(&payload, "unused")), None); + assert_eq!(default_of(&messages), None); } - assert_eq!(default_title(&serde_json::json!({"text": "No role"})), None); + assert_eq!(default_title(&json!({"text":"No role"})), None); } #[test] - fn other_agents_keep_the_default_session_strategy() { - let text = format!("{CLAUDE_CODE_NAMING_INSTRUCTION}\n{CODEX_NAMING_INSTRUCTION}"); - for agent in ["cursor", "openai", "unknown", ""] { - assert!(NamingRequest::parse(agent, Some(&text), &Value::Null).is_none()); - } + fn agent_titles_come_only_from_the_collector_field() { assert_eq!( - TitleTask::from_request("codex", CLAUDE_CODE_NAMING_INSTRUCTION), - None + TitleContribution::of( + "response", + &json!({"purpose":"session_title","session_title":"登录修复"}) + ), + Some(TitleContribution::Agent("登录修复")) ); + // A reply that merely looks like a title object is ordinary output. assert_eq!( - TitleTask::from_request("claude-code", CODEX_NAMING_INSTRUCTION), + TitleContribution::of( + "response", + &request(&json!({"messages":[ + {"role":"assistant","text":"{\"title\":\"Quoted\"}"} + ]})) + ), None ); } #[test] - fn naming_reads_protocol_roles_and_preserves_nested_prompt_markers() { - let text = format!("{CODEX_NAMING_INSTRUCTION}\nUser prompt:\nExplain User prompt:\nhello"); - let mut payload = serde_json::json!({ - "input": [{"role": "user", "content": [{"type": "input_text", "text": text}]}], - "client_metadata": {"thread_source": "thread_title"} - }); - let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); - assert_eq!( - NamingRequest::parse("codex", None, &metadata) - .expect("naming") - .prompt - .as_deref(), - Some("Explain User prompt:\nhello") - ); - payload["client_metadata"]["thread_source"] = serde_json::json!("user"); - let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); - assert!( - NamingRequest::parse("codex", Some(&text), &metadata).is_none(), - "a user quoting instructions is not a title thread" - ); - payload["client_metadata"]["thread_source"] = serde_json::json!("thread_title"); - payload["input"][0]["role"] = serde_json::json!("assistant"); - let metadata = archived_request(&payload, "01a0f320-138e-7d21-99ea-03a7ae2a9b50"); - assert!( - NamingRequest::parse("codex", Some(&text), &metadata).is_none(), - "do not fall back to flattened quoted text" - ); + fn naming_requests_are_not_user_input() { + let mut metadata = request(&json!({"messages":[{"role":"user","text":"…"}]})); + metadata["purpose"] = json!("session_title"); + assert_eq!(TitleContribution::of("request", &metadata), None); } #[test] - fn claude_names_its_own_session_from_raw_instructions() { - let metadata = archived_request( - &serde_json::json!({ - "system": CLAUDE_CODE_NAMING_INSTRUCTION, - "messages": [{"role": "user", "content": "Fix the build"}] - }), - "unused", + fn codex_naming_threads_are_marked_from_their_thread_source() { + let messages = json!([{"role":"user","text":"Generate a concise title"}]); + assert_eq!( + TitleContribution::of( + "request", + &request(&json!({"messages":messages,"thread_source":"thread_title"})) + ), + Some(TitleContribution::TitleThread) ); assert_eq!( - NamingRequest::parse("claude-code", None, &metadata) - .expect("CC naming") - .task, - TitleTask::NamesOwnSession + TitleContribution::of( + "request", + &request(&json!({"messages":messages,"thread_source":"user"})) + ), + Some(TitleContribution::UserInput) ); } #[test] - fn incomplete_raw_bodies_do_not_fall_back_to_legacy_text() { - for part in [ - serde_json::json!({"truncated": true, "content_base64": "e30="}), - serde_json::json!({"truncated": false, "content_base64": "invalid"}), - serde_json::json!({"truncated": false, "content_encoding": "gzip", "content_base64": "e30="}), - ] { - let metadata = serde_json::json!({"raw_body": {"parts": [part]}, - "conversation": {"messages": [{"role": "user", "text": "hello"}]}}); - assert!(!opens_with_prompt(&metadata, "hello")); - assert_eq!(default_title(&metadata), None); - assert!( - NamingRequest::parse("codex", Some(CODEX_NAMING_INSTRUCTION), &metadata).is_none() + fn incomplete_conversations_give_no_default() { + for status in ["partial", "unsupported"] { + let metadata = request( + &json!({"completeness":status,"messages":[{"role":"user","text":"hello"}]}), ); + assert_eq!(default_title(&metadata), None); } } + + #[test] + fn content_outside_collector_messages_is_never_interpreted() { + let metadata = json!({"raw_body":{"parts":[{"content_base64":"e30="}]}, + "conversation":{"messages":[], "completeness":"complete", "provider":{"input":"secret"}}}); + assert_eq!(default_title(&metadata), None); + } } diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index af75eb4..5d72b9a 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -5,9 +5,9 @@ //! role-aware messages, first-seen tool results, and response text and tool //! calls, with each tool call paired to its result by call identifier. -mod raw_body; +mod conversation; -use raw_body::RawProjection; +use conversation::ConversationProjection; use std::{ collections::{HashMap, HashSet, hash_map::DefaultHasher}, @@ -230,7 +230,7 @@ pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { #[derive(Default)] struct Merger { - raw: RawProjection, + conversation: ConversationProjection, items: Vec, seen_results: HashSet, streams: HashMap, @@ -248,9 +248,9 @@ impl Merger { fn request(&mut self, event: &UsageEventResponse) { self.pending_compaction = None; let first_new = self.items.len(); - if let Some(items) = self - .raw - .project(event, &mut self.current_stream, &mut self.streams) + if let Some(items) = + self.conversation + .project(event, &mut self.current_stream, &mut self.streams) { self.items.extend(items); } else { @@ -362,9 +362,9 @@ impl Merger { fn response(&mut self, event: &UsageEventResponse) { let stream = self.current_stream; - let projected = self - .raw - .project(event, &mut self.current_stream, &mut self.streams); + let projected = + self.conversation + .project(event, &mut self.current_stream, &mut self.streams); if let Some(index) = self.pending_compaction.take() { // The response to the compaction prompt is the handoff summary. if let Some(summary) = event.text.as_deref().map(str::trim) @@ -431,13 +431,13 @@ fn segment_str<'a>(segment: &'a Value, key: &str) -> Option<&'a str> { mod tests { #[test] fn usage_retains_events_without_new_content_and_counts_each_event_once() { - let body = r#"{"input":[{"role":"user","content":"hello"}]}"#; - let mut request = raw_event("request", body); + let body = r#"{"messages":[{"id":null,"role":"user","text":"hello"}],"instructions_present":false,"previous_response_id":null}"#; + let mut request = structured_event("request", body); request.input_tokens = 2; request.cache_read_tokens = 77_000; request.cache_write_tokens = 400; request.total_tokens = 77_402; - let repeated = raw_event("request", body); + let repeated = structured_event("request", body); let mut response = event("response", "", json!({"token_usage_estimated": true})); response.llm_model = "other-model".to_owned(); response.output_tokens = 150; @@ -580,7 +580,7 @@ mod tests { } fn user_request(texts: &[&str]) -> UsageEventResponse { - raw_event("request", &json!({"input": texts.iter().map(|text| json!({"role":"user", "content":text})).collect::>()}).to_string()) + structured_event("request", &json!({"messages": texts.iter().map(|text| json!({"role":"user", "text":text})).collect::>()}).to_string()) } #[test] @@ -588,8 +588,11 @@ mod tests { let text = "system: instructions\n\n用户内容\n\n用户内容"; for metadata in [ json!({}), - json!({"raw_body":{"parts":[{"truncated":true}]}}), - json!({"raw_body":{"parts":[{"truncated":false,"content_base64":"invalid"}]}}), + json!({"conversation":{"completeness":"partial","messages":[]}}), + json!({"conversation":{"messages":"invalid"}}), + json!({"conversation":{"messages":[],"completeness":"complete"}}), + // Older collectors stored messages without completeness. + json!({"conversation":{"messages":[{"role":"user","text":"用户内容"}]}}), ] { let items = build(timeline(vec![event("request", text, metadata)])).items; assert_eq!(items.len(), 1); @@ -597,7 +600,7 @@ mod tests { assert_eq!(items[0].text.as_deref(), Some(text)); assert!(!items[0].continued); } - let items = build(timeline(vec![raw_event("request", r#"{"input":[{"content":"system: unknown"},{"role":"unrecognized","content":"hello"},{"role":"user","content":"system: quoted text"}]}"#)])).items; + let items = build(timeline(vec![structured_event("request", r#"{"messages":[{"id":null,"role":"unknown","text":"system: unknown"},{"id":null,"role":"unrecognized","text":"hello"},{"id":null,"role":"user","text":"system: quoted text"}],"instructions_present":false,"previous_response_id":null}"#)])).items; assert_eq!( items.iter().map(|i| i.kind).collect::>(), [ @@ -608,28 +611,31 @@ mod tests { ); } - fn raw_event(side: &str, body: &str) -> UsageEventResponse { - use base64::{Engine as _, engine::general_purpose::STANDARD}; + /// A current collector always declares completeness; default it to complete. + fn structured_event(side: &str, body: &str) -> UsageEventResponse { + let mut conversation = serde_json::from_str::(body).unwrap(); + conversation + .as_object_mut() + .unwrap() + .entry("completeness") + .or_insert_with(|| json!("complete")); event( side, "legacy fallback", - json!({"raw_body": {"transport": "http", "parts": [{ - "content_base64": STANDARD.encode(body.as_bytes()), "truncated": false, - "byte_size": body.len(), "content_type": "application/json" - }]}}), + json!({"conversation": conversation}), ) } #[test] - fn raw_archive_projection_keeps_roles_and_repeated_user_occurrences() { - let first = r#" {"instructions":"system instructions","input":[{"role":"user","content":"继续"}],"unknown":[1,1]} "#; - let response = r#"{"output":[{"id":"a1","role":"assistant","content":[{"type":"output_text","text":"working"}]}]}"#; - let replay = r#"{"instructions":"system instructions","input":[{"role":"user","content":"继续"},{"id":"a1","role":"assistant","content":[{"type":"output_text","text":"working"}]},{"role":"user","content":"继续"}]}"#; + fn structured_projection_keeps_roles_and_repeated_user_occurrences() { + let first = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true,"previous_response_id":null}"#; + let response = r#"{"messages":[{"id":"a1","role":"assistant","text":"working"}],"instructions_present":false,"previous_response_id":null}"#; + let replay = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"},{"id":"a1","role":"assistant","text":"working"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true,"previous_response_id":null}"#; let events = vec![ - raw_event("request", first), - raw_event("response", response), - raw_event("request", replay), - raw_event("request", replay), + structured_event("request", first), + structured_event("response", response), + structured_event("request", replay), + structured_event("request", replay), ]; let metadata_before = events .iter() @@ -665,11 +671,11 @@ mod tests { #[test] fn streamed_anthropic_response_is_not_replayed_as_user_input() { - let response = "event: content_block_start\r\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\r\n\r\nevent: content_block_delta\r\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi\"}}\r\n\r\n"; + let response = r#"{"messages":[{"role":"assistant","text":"hi"}]}"#; let items = build(timeline(vec![ - raw_event("request", r#"{"system":"instructions","messages":[{"role":"user","content":"hello"}]}"#), - raw_event("response", response), - raw_event("request", r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"assistant","content":"hi"},{"role":"user","content":"hello"}]}"#), + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#), + structured_event("response", response), + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#), ])).items; assert_eq!( items.iter().map(|i| i.kind).collect::>(), @@ -683,17 +689,44 @@ mod tests { assert_eq!(items[2].text.as_deref(), Some("hi")); } + #[test] + fn anthropic_reasoning_is_shown_once_with_its_assistant_message() { + let first = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null,"completeness":"complete"}"#; + let response = r#"{"messages":[{"id":"msg_1","role":"reasoning","text":"thinking it over"},{"id":"msg_1","role":"assistant","text":"hi"}],"instructions_present":false,"previous_response_id":null,"completeness":"complete"}"#; + let replay = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"reasoning","text":"thinking it over"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"previous_response_id":null,"completeness":"complete"}"#; + let items = build(timeline(vec![ + structured_event("request", first), + structured_event("response", response), + structured_event("request", replay), + ])) + .items; + assert_eq!( + items + .iter() + .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + (TrajectoryItemKind::Context, "instructions"), + (TrajectoryItemKind::User, "hello"), + (TrajectoryItemKind::Assistant, "thinking it over"), + (TrajectoryItemKind::Assistant, "hi"), + (TrajectoryItemKind::User, "next"), + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + #[test] fn incremental_requests_keep_identical_new_user_messages() { let items = build(timeline(vec![ - raw_event( + structured_event( "request", - r#"{"instructions":"instructions","input":[{"role":"user","content":"continue"}]}"#, + r#"{"messages":[{"role":"developer","text":"instructions"},{"id":null,"role":"user","text":"continue"}],"instructions_present":true,"previous_response_id":null}"#, ), - raw_event("response", r#"{"output_text":"working"}"#), - raw_event( + structured_event("response", r#"{"messages":[{"id":null,"role":"assistant","text":"working"}],"instructions_present":false,"previous_response_id":null}"#), + structured_event( "request", - r#"{"previous_response_id":"r1","input":[{"role":"user","content":"continue"}]}"#, + r#"{"messages":[{"id":null,"role":"user","text":"continue"}],"instructions_present":false,"previous_response_id":"r1"}"#, ), ])) .items; @@ -710,11 +743,11 @@ mod tests { } #[test] - fn raw_anthropic_request_retains_system_and_assistant_roles() { - let body = r#"{"system":[{"type":"text","text":"instructions"}],"messages":[{"role":"user","content":"hello"},{"role":"assistant","content":[{"type":"text","text":"hi"}]},{"role":"user","content":"hello"}]}"#; + fn structured_request_retains_system_and_assistant_roles() { + let body = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#; let items = build(timeline(vec![ - raw_event("request", body), - raw_event("request", body), + structured_event("request", body), + structured_event("request", body), ])) .items; assert_eq!( @@ -731,22 +764,22 @@ mod tests { #[test] fn changing_billing_headers_and_token_budgets_keep_one_stream() { let request = |hash: &str, left: u32, messages: &str| { - format!( - r#"{{"system":[{{"type":"text","text":"x-anthropic-billing-header: cc_version=2.1.288; cch={hash}; cc_prev_req=req_{hash};"}},{{"type":"text","text":"instructions\n\n{left} tokens left"}}],"messages":{messages}}}"# - ) + let mut messages: Vec = serde_json::from_str(messages).unwrap(); + messages.insert(0, json!({"role":"system", "text": format!("x-anthropic-billing-header: cc_version=2.1.288; cch={hash}; cc_prev_req=req_{hash};\ninstructions\n\n{left} tokens left")})); + json!({"messages": messages, "instructions_present": true}).to_string() }; let items = build(timeline(vec![ - raw_event( + structured_event( "request", - &request("a1", 900, r#"[{"role":"user","content":"hello"}]"#), + &request("a1", 900, r#"[{"role":"user","text":"hello"}]"#), ), - raw_event("response", r#"{"content":[{"type":"text","text":"hi"}]}"#), - raw_event( + structured_event("response", r#"{"messages":[{"id":null,"role":"assistant","text":"hi"}],"instructions_present":false,"previous_response_id":null}"#), + structured_event( "request", &request( "b2", 850, - r#"[{"role":"user","content":"hello"},{"role":"assistant","content":"hi"},{"role":"user","content":"next"}]"#, + r#"[{"role":"user","text":"hello"},{"role":"assistant","text":"hi"},{"role":"user","text":"next"}]"#, ), ), ])) @@ -770,13 +803,253 @@ mod tests { assert!(items.iter().all(|i| i.stream == 0)); } + /// Claude Code thread requests as the collector projects them: a + /// `create` carries the whole system prompt and history, a `continue` + /// only the billing header and the messages added since + /// `previous_message_id`. + fn claude_request(thread: Value, system: &[&str], messages: Value) -> UsageEventResponse { + let system = std::iter::once( + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; cch=3ceb5;", + ) + .chain(system.iter().copied()) + .collect::>() + .join("\n"); + let mut conversation = json!({ + "messages": [{"id": null, "role": "system", "text": system}], + "instructions_present": true, + "previous_response_id": null, + }); + conversation["messages"] + .as_array_mut() + .unwrap() + .extend(serde_json::from_value::>(messages).unwrap()); + let mut request = structured_event("request", &conversation.to_string()); + if !thread.is_null() { + request.metadata["claude_thread"] = thread; + } + request + } + + fn create(messages: Value) -> UsageEventResponse { + claude_request( + json!({"type": "create"}), + &["You are Claude Code.", "Main loop instructions."], + messages, + ) + } + + fn continue_from(previous: &str, messages: Value) -> UsageEventResponse { + claude_request( + json!({"type": "continue", "previous_message_id": previous}), + &[], + messages, + ) + } + + fn user(text: &str) -> Value { + json!({"id": null, "role": "user", "text": text}) + } + + fn budget() -> Value { + json!({"id": null, "role": "system", "text": "15000000 tokens left"}) + } + + fn tool_result() -> Value { + json!({"id": null, "role": "tool", "text": ""}) + } + + /// The replayed form of a response: thinking is redacted. + fn replayed(text: &str) -> [Value; 3] { + [ + json!({"id": null, "role": "reasoning", "text": ""}), + json!({"id": null, "role": "assistant", "text": text}), + json!({"id": null, "role": "tool", "text": ""}), + ] + } + + fn streamed(id: &str, text: &str) -> UsageEventResponse { + let conversation = json!({ + "messages": [ + {"id": id, "role": "reasoning", "text": ""}, + {"id": id, "role": "assistant", "text": text}, + {"id": null, "role": "tool", "text": ""}, + ], + "instructions_present": false, + "previous_response_id": null, + }); + structured_event("response", &conversation.to_string()) + } + + /// A `thread_not_found` error carries no messages. + fn thread_not_found() -> UsageEventResponse { + let mut response = structured_event( + "response", + r#"{"messages":[],"instructions_present":false,"previous_response_id":null,"completeness":"unsupported"}"#, + ); + response.text = None; + response + } + + fn texts(items: &[TrajectoryItem]) -> Vec<(TrajectoryItemKind, usize, &str)> { + items + .iter() + .map(|i| (i.kind, i.stream, i.text.as_deref().unwrap_or_default())) + .collect() + } + + const MAIN_CONTEXT: &str = "x-anthropic-billing-header: cc_version=2.1.293.af3; \ + cc_entrypoint=cli; cch=3ceb5;\nYou are Claude Code.\nMain loop \ + instructions."; + + #[test] + fn thread_continue_appends_new_messages_in_the_stream_it_continues() { + let search = claude_request( + Value::Null, + &[ + "You are Claude Code.", + "You are an assistant for performing a web search tool use", + ], + json!([user("search the web")]), + ); + let events = vec![ + create(json!([user("hello"), budget()])), + streamed("msg_1", "looking"), + search, + streamed("msg_side", "results"), + continue_from("msg_1", json!([tool_result(), budget()])), + streamed("msg_2", "found it"), + continue_from("msg_2", json!([tool_result(), user("thanks"), budget()])), + ]; + let items = build(timeline(events)).items; + + assert_eq!( + texts(&items), + [ + (TrajectoryItemKind::Context, 0, MAIN_CONTEXT), + (TrajectoryItemKind::User, 0, "hello"), + ( + TrajectoryItemKind::Context, + 0, + "15000000 tokens left" + ), + (TrajectoryItemKind::Assistant, 0, "looking"), + ( + TrajectoryItemKind::Context, + 1, + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ + cch=3ceb5;\nYou are Claude Code.\nYou are an assistant for performing a web \ + search tool use" + ), + (TrajectoryItemKind::User, 1, "search the web"), + (TrajectoryItemKind::Assistant, 1, "results"), + (TrajectoryItemKind::Assistant, 0, "found it"), + (TrajectoryItemKind::User, 0, "thanks"), + ] + ); + } + + #[test] + fn thread_replay_after_a_failed_continue_adds_no_duplicates() { + let first = user("hello"); + let retry = user("try the wire"); + let [looking_thinking, looking, looking_tool] = replayed("looking"); + let [found_thinking, found, found_tool] = replayed("found it"); + let replay = json!([ + first, + budget(), + looking_thinking, + looking, + looking_tool, + tool_result(), + budget(), + found_thinking, + found, + found_tool, + retry, + budget(), + ]); + let events = vec![ + create(json!([first, budget()])), + streamed("msg_1", "looking"), + continue_from("msg_1", json!([tool_result(), budget()])), + streamed("msg_2", "found it"), + continue_from("msg_2", json!([retry, budget()])), + thread_not_found(), + create(replay), + streamed("msg_3", "no wire port"), + continue_from("msg_3", json!([tool_result(), budget()])), + streamed("msg_4", "compressed"), + ]; + let trajectory = build(timeline(events)); + + assert_eq!( + texts(&trajectory.items), + [ + (TrajectoryItemKind::Context, 0, MAIN_CONTEXT), + (TrajectoryItemKind::User, 0, "hello"), + ( + TrajectoryItemKind::Context, + 0, + "15000000 tokens left" + ), + (TrajectoryItemKind::Assistant, 0, "looking"), + (TrajectoryItemKind::Assistant, 0, "found it"), + (TrajectoryItemKind::User, 0, "try the wire"), + (TrajectoryItemKind::Assistant, 0, "no wire port"), + (TrajectoryItemKind::Assistant, 0, "compressed"), + ] + ); + assert_eq!(trajectory.events.len(), 10); + } + + #[test] + fn thread_continue_of_an_unknown_message_follows_the_latest_thread() { + let events = vec![ + create(json!([user("hello"), budget()])), + claude_request( + Value::Null, + &["You are a title generator."], + json!([user("name it")]), + ), + continue_from("msg_uncaptured", json!([user("next"), budget()])), + ]; + let items = build(timeline(events)).items; + assert_eq!( + items.last().map(|i| (i.kind, i.stream, i.text.as_deref())), + Some((TrajectoryItemKind::User, 0, Some("next"))) + ); + + // Without any earlier thread request the continue keeps its own stream. + let items = build(timeline(vec![ + claude_request( + Value::Null, + &["You are a title generator."], + json!([user("name it")]), + ), + continue_from("msg_uncaptured", json!([user("next")])), + ])) + .items; + assert_eq!( + texts(&items)[2..], + [ + ( + TrajectoryItemKind::Context, + 1, + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ + cch=3ceb5;" + ), + (TrajectoryItemKind::User, 1, "next"), + ] + ); + } + #[test] fn system_messages_inside_the_conversation_keep_one_stream() { - let first = r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"system","content":"reminder one"}]}"#; - let second = r#"{"system":"instructions","messages":[{"role":"user","content":"hello"},{"role":"system","content":"reminder one"},{"role":"assistant","content":"hi"},{"role":"system","content":"reminder two"},{"role":"user","content":"next"}]}"#; + let first = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"}],"instructions_present":true,"previous_response_id":null}"#; + let second = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"system","text":"reminder two"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"previous_response_id":null}"#; let items = build(timeline(vec![ - raw_event("request", first), - raw_event("request", second), + structured_event("request", first), + structured_event("request", second), ])) .items; @@ -798,13 +1071,13 @@ mod tests { } #[test] - fn raw_compaction_keeps_existing_summary_presentation() { - let request = json!({"input": [{"role": "user", "content": "write report"}, {"role": "user", "content": PROMPT}]}).to_string(); - let replay = json!({"input": [{"role": "user", "content": "write report"}, {"role": "user", "content": handoff()}]}).to_string(); + fn structured_compaction_keeps_existing_summary_presentation() { + let request = json!({"messages": [{"role": "user", "text": "write report"}, {"role": "user", "text": PROMPT}]}).to_string(); + let replay = json!({"messages": [{"role": "user", "text": "write report"}, {"role": "user", "text": handoff()}]}).to_string(); let items = build(timeline(vec![ - raw_event("request", &request), + structured_event("request", &request), event("response", SUMMARY, json!({})), - raw_event("request", &replay), + structured_event("request", &replay), ])) .items; assert_eq!( diff --git a/crates/abyss-backend/src/usage/trajectory/conversation.rs b/crates/abyss-backend/src/usage/trajectory/conversation.rs new file mode 100644 index 0000000..d15b6b2 --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory/conversation.rs @@ -0,0 +1,229 @@ +//! Session trajectory projection from collector-produced structured messages. + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; +use crate::usage::conversation::{Conversation, Message, Role}; + +impl Message { + fn matches(&self, other: &Self) -> bool { + self.role == other.role + && self.text == other.text + && match (&self.id, &other.id) { + (Some(left), Some(right)) => left == right, + _ => true, + } + } + + const fn kind(&self) -> TrajectoryItemKind { + match self.role { + Role::System | Role::Developer => TrajectoryItemKind::Context, + Role::User => TrajectoryItemKind::User, + Role::Assistant | Role::Reasoning => TrajectoryItemKind::Assistant, + Role::Tool | Role::Unknown => TrajectoryItemKind::Request, + } + } +} + +#[derive(Default)] +pub(super) struct ConversationProjection { + streams: HashMap, + /// Stream of each response by its provider message id, so an Anthropic + /// thread `continue` naming that message joins the stream it continues. + messages: HashMap, + /// Stream of the latest Anthropic thread request. + thread: Option, +} + +#[derive(Default)] +struct StreamHistory { + history: Vec, + contexts: HashSet<(Role, String)>, +} + +impl ConversationProjection { + pub(super) fn project( + &mut self, + event: &UsageEventResponse, + stream: &mut usize, + stream_ids: &mut HashMap, + ) -> Option> { + // Older collectors stored messages without completeness; their events keep + // the flattened-text presentation they had before structured conversations. + if !Conversation::declared(&event.metadata) { + return None; + } + let conversation = Conversation::read(&event.metadata)?; + let request = event.event_type == "request"; + let (messages, incremental) = if request { + let messages = conversation + .messages + .into_iter() + .filter(|m| m.role != Role::Tool) + .collect::>(); + if messages.is_empty() { + return None; + } + let incremental = self.request_stream( + event, + conversation.previous_response_id.is_some(), + conversation.instructions_present, + &messages, + stream, + stream_ids, + ); + (messages, incremental) + } else { + if let Some(id) = conversation.messages.iter().find_map(|m| m.id.as_deref()) { + self.messages.insert(id.to_owned(), *stream); + } + let mut messages = conversation + .messages + .into_iter() + .filter(|m| m.role != Role::Tool) + .collect::>(); + if messages.is_empty() + && let Some(text) = event.text.as_ref().filter(|v| !v.is_empty()) + { + messages.push(Message { + id: None, + role: Role::Assistant, + text: text.clone(), + }); + } + (messages, false) + }; + // Legacy response text remains available when no message was projected. + if !request && messages.is_empty() { + return None; + } + let state = self.streams.entry(*stream).or_default(); + let mut cursor = 0; + let mut next = Vec::new(); + let mut added = Vec::new(); + for message in messages { + let context = message.kind() == TrajectoryItemKind::Context; + let known = if context { + !state + .contexts + .insert((message.role, stable_context(&message.text))) + } else if request && !incremental { + let found = state.history[cursor..] + .iter() + .position(|old| old.matches(&message)); + if let Some(offset) = found { + cursor = cursor.saturating_add(offset).saturating_add(1); + } + found.is_some() + } else if request { + state.history.iter().any(|old| { + old.matches(&message) && (message.id.is_some() || message.role != Role::User) + }) + } else { + false + }; + if !context { + if request && !incremental { + next.push(message.clone()); + } else if !known { + state.history.push(message.clone()); + } + } + if !known && !message.text.is_empty() { + let mut item = TrajectoryItem::new(message.kind(), *stream, event); + item.text = Some(message.text); + added.push(item); + } + } + if request && !incremental { + state.history = next; + } + Some(added) + } + + /// Points `stream` at the stream this request belongs to and tells + /// whether the request sends only messages added since an earlier one. + fn request_stream( + &mut self, + event: &UsageEventResponse, + responds_to_previous: bool, + instructions_present: bool, + messages: &[Message], + stream: &mut usize, + stream_ids: &mut HashMap, + ) -> bool { + // An Anthropic thread `continue` sends only the messages added + // since the response it names, and a system prompt that is only + // the billing header, so it neither repeats nor names its stream. + let thread = event.metadata.get("claude_thread"); + let continued = thread + .and_then(|thread| thread.get("type")) + .and_then(Value::as_str) + == Some("continue"); + let incremental = continued || responds_to_previous; + let continued_stream = continued + .then(|| { + thread + .and_then(|thread| thread.get("previous_message_id")) + .and_then(Value::as_str) + .and_then(|id| self.messages.get(id).copied()) + // The named response was not captured: thread + // requests of a session follow one another. + .or(self.thread) + }) + .flatten(); + if let Some(continued) = continued_stream { + *stream = continued; + } else if !incremental || instructions_present { + // A stream is named by the prompt the conversation opens with. + // System messages later in the conversation are part of it: + // counting them made each new one start another stream. + let context = messages + .iter() + .take_while(|message| message.kind() == TrajectoryItemKind::Context) + .map(|message| { + format!( + "{}: {}", + message.role.as_str(), + stable_context(&message.text) + ) + }) + .collect::>() + .join("\n\n"); + let next = stream_ids.len(); + *stream = *stream_ids.entry(super::hash_str(&context)).or_insert(next); + } + if thread.is_some() { + self.thread = Some(*stream); + } + incremental + } +} + +/// Claude Code adds two lines to its system prompt that change on every +/// request: a leading billing header whose request hash and previous-request +/// id differ each time, for example `x-anthropic-billing-header: +/// cc_version=2.1.288.9e5; cc_entrypoint=cli; cch=e4f32; cc_is_subagent=true; +/// cc_prev_req=req_…; cc_prompt_id=…;`, and a trailing context budget such as +/// `14964936 tokens left`. Counting them made +/// every request a new stream that replayed the whole conversation, so +/// streams and system prompts are told apart without them. The stored conversation +/// keeps both, so they can still be counted for billing. +const BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:"; +const TOKEN_BUDGET_OPEN: &str = ""; +const TOKEN_BUDGET_CLOSE: &str = ""; + +/// A system prompt without the lines that change on every request. +fn stable_context(text: &str) -> String { + text.lines() + .filter(|line| !changes_every_request(line.trim())) + .collect::>() + .join("\n") +} + +fn changes_every_request(line: &str) -> bool { + line.starts_with(BILLING_HEADER_PREFIX) + || (line.starts_with(TOKEN_BUDGET_OPEN) && line.ends_with(TOKEN_BUDGET_CLOSE)) +} diff --git a/crates/abyss-backend/src/usage/trajectory/raw_body.rs b/crates/abyss-backend/src/usage/trajectory/raw_body.rs deleted file mode 100644 index 6548f15..0000000 --- a/crates/abyss-backend/src/usage/trajectory/raw_body.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Read-only projection of archived provider bodies; the archive is never edited. - -use std::collections::{BTreeMap, HashMap, HashSet}; - -use serde_json::Value; - -use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; -use crate::usage::provider_body::{ - Message, bodies, message, request_messages, request_payload, text, -}; - -impl Message { - fn matches(&self, other: &Self) -> bool { - self.role == other.role - && self.text == other.text - && match (&self.id, &other.id) { - (Some(left), Some(right)) => left == right, - _ => true, - } - } - - fn kind(&self) -> TrajectoryItemKind { - match self.role.as_str() { - "system" | "developer" => TrajectoryItemKind::Context, - "user" => TrajectoryItemKind::User, - "assistant" | "reasoning" => TrajectoryItemKind::Assistant, - _ => TrajectoryItemKind::Request, - } - } -} - -#[derive(Default)] -pub(super) struct RawProjection { - streams: HashMap, -} - -#[derive(Default)] -struct StreamHistory { - history: Vec, - contexts: HashSet<(String, String)>, -} - -impl RawProjection { - pub(super) fn project( - &mut self, - event: &UsageEventResponse, - stream: &mut usize, - stream_ids: &mut HashMap, - ) -> Option> { - let values = bodies(&event.metadata)?; - let request = event.event_type == "request"; - let (messages, incremental) = if request { - let payload = request_payload(&values)?; - let messages = request_messages(payload); - let incremental = payload - .get("previous_response_id") - .is_some_and(Value::is_string); - if !incremental - || payload.get("instructions").is_some() - || payload.get("system").is_some() - { - // A stream is named by the prompt the conversation opens with. - // System messages later in the conversation are part of it: - // counting them made each new one start another stream. - let context = messages - .iter() - .take_while(|message| message.kind() == TrajectoryItemKind::Context) - .map(|message| format!("{}: {}", message.role, stable_context(&message.text))) - .collect::>() - .join("\n\n"); - let next = stream_ids.len(); - *stream = *stream_ids.entry(super::hash_str(&context)).or_insert(next); - } - (messages, incremental) - } else { - let mut messages = response_messages(&values); - if messages.is_empty() - && let Some(text) = event.text.as_ref().filter(|v| !v.is_empty()) - { - messages.push(Message { - id: None, - role: "assistant".to_owned(), - text: text.clone(), - }); - } - (messages, false) - }; - // Incomplete/unrecognized response formats still have the older parsed - // text projection. Archival success must not depend on this parser. - if !request && messages.is_empty() { - return None; - } - let state = self.streams.entry(*stream).or_default(); - let mut cursor = 0; - let mut next = Vec::new(); - let mut added = Vec::new(); - for message in messages { - let context = message.kind() == TrajectoryItemKind::Context; - let known = if context { - !state - .contexts - .insert((message.role.clone(), stable_context(&message.text))) - } else if request && !incremental { - let found = state.history[cursor..] - .iter() - .position(|old| old.matches(&message)); - if let Some(offset) = found { - cursor = cursor.saturating_add(offset).saturating_add(1); - } - found.is_some() - } else if request { - state.history.iter().any(|old| { - old.matches(&message) && (message.id.is_some() || message.role != "user") - }) - } else { - false - }; - if !context { - if request && !incremental { - next.push(message.clone()); - } else if !known { - state.history.push(message.clone()); - } - } - if !known && !message.text.is_empty() { - let mut item = TrajectoryItem::new(message.kind(), *stream, event); - item.text = Some(message.text); - added.push(item); - } - } - if request && !incremental { - state.history = next; - } - Some(added) - } -} - -/// Claude Code adds two lines to its system prompt that change on every -/// request: a leading billing header whose request hash and previous-request -/// id differ each time, for example `x-anthropic-billing-header: -/// cc_version=2.1.288.9e5; cc_entrypoint=cli; cch=e4f32; cc_is_subagent=true; -/// cc_prev_req=req_…; cc_prompt_id=…;`, and a trailing context budget such as -/// `14964936 tokens left`. Counting them made -/// every request a new stream that replayed the whole conversation, so -/// streams and system prompts are told apart without them. The archived body -/// keeps both, so they can still be counted for billing. -const BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:"; -const TOKEN_BUDGET_OPEN: &str = ""; -const TOKEN_BUDGET_CLOSE: &str = ""; - -/// A system prompt without the lines that change on every request. -fn stable_context(text: &str) -> String { - text.lines() - .filter(|line| !changes_every_request(line.trim())) - .collect::>() - .join("\n") -} - -fn changes_every_request(line: &str) -> bool { - line.starts_with(BILLING_HEADER_PREFIX) - || (line.starts_with(TOKEN_BUDGET_OPEN) && line.ends_with(TOKEN_BUDGET_CLOSE)) -} - -fn response_messages(values: &[Value]) -> Vec { - let mut items = BTreeMap::new(); - let mut anthropic_blocks = BTreeMap::::new(); - let mut chat_text = String::new(); - for value in values { - let kind = value - .get("type") - .and_then(Value::as_str) - .unwrap_or_default(); - let index = value.get("index").and_then(Value::as_u64).unwrap_or(0); - if kind == "content_block_start" { - if let Some(block) = value.get("content_block") { - anthropic_blocks.insert(index, text(block)); - } - } else if kind == "content_block_delta" - && let Some(delta) = value.get("delta") - { - anthropic_blocks - .entry(index) - .or_default() - .push_str(&text(delta)); - } - if let Some(delta) = value - .pointer("/choices/0/delta/content") - .and_then(Value::as_str) - { - chat_text.push_str(delta); - } - let response = value.get("response").unwrap_or(value); - if let Some(output) = response.get("output").and_then(Value::as_array) { - for (index, value) in output.iter().enumerate() { - if let Some(message) = message(value, "assistant") { - items.insert(u64::try_from(index).unwrap_or(u64::MAX), message); - } - } - } else if value.get("type").and_then(Value::as_str) == Some("response.output_item.done") { - if let Some(message) = value.get("item").and_then(|v| message(v, "assistant")) { - items.insert( - value - .get("output_index") - .and_then(Value::as_u64) - .unwrap_or(0), - message, - ); - } - } else if let Some(message) = value - .get("choices") - .and_then(Value::as_array) - .and_then(|v| v.first()) - .and_then(|v| v.get("message")) - .and_then(|v| message(v, "assistant")) - { - items.insert(0, message); - } else if (value.get("content").is_some() || value.get("output_text").is_some()) - && let Some(message) = message(value, "assistant") - { - items.insert(0, message); - } - } - if items.is_empty() { - let text = if anthropic_blocks.is_empty() { - chat_text - } else { - anthropic_blocks - .into_values() - .filter(|s| !s.is_empty()) - .collect::>() - .join("\n") - }; - if !text.is_empty() { - items.insert( - 0, - Message { - id: None, - role: "assistant".to_owned(), - text, - }, - ); - } - } - items.into_values().collect() -} diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index 9d9b533..00c6074 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -491,6 +491,10 @@ def run(base_url: str, token: str, run_id: str) -> None: from session_titles_api import run_title_contract run_title_contract(base_url, token, run_id) + + from claude_turns_api import run_claude_turn_contract + + run_claude_turn_contract(base_url, token, run_id) print(f"blackbox API contract passed for {session_id}") diff --git a/scripts/tests/claude_turns_api.py b/scripts/tests/claude_turns_api.py new file mode 100644 index 0000000..b5ef7cd --- /dev/null +++ b/scripts/tests/claude_turns_api.py @@ -0,0 +1,76 @@ +"""Claude Code turn attachment contract shared by both storage black-box suites.""" + +import urllib.parse + +from blackbox_api import event, request, require, require_status + + +def run_claude_turn_contract(base_url: str, token: str, run_id: str) -> None: + session_id = f"{run_id}-claude-thread" + + def make(name, second, metadata, *, agent="claude-code", session=session_id): + value = event(f"{run_id}-{name}", session, "request", name, {"input_tokens": 1}, run_id) + value.pop("attachments", None) + value["agent"]["name"] = agent + value["llm"] = {"provider": "Anthropic", "model": "claude-x"} + value["observed_at"] = f"2026-10-08T00:00:{second:02}Z" + # Current collectors send no turn number; blackbox_api's events still + # carry one, as older collectors do. + value.pop("turn_index") + value["metadata"] = metadata + return value + + def main_loop(turn, prompt): + return { + "claude_turn_id": f"cc_turn:{turn}", + "claude_code_billing": {"cc_prompt_id": prompt, "cc_turn_index": str(turn)}, + } + + def ingest(*events): + result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) + require_status(result, 200, "Claude turn fixture ingest") + + def turns(session): + query = urllib.parse.urlencode({"session_id": session, "fields": "list", "limit": 100}) + result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) + require_status(result, 200, "Claude turn event lookup") + events = result.json()["events"] + return {item["event_id"].removeprefix(f"{run_id}-"): item["turn_pk"] for item in events} + + ingest( + make("main-1", 10, main_loop(1, "prompt-1")), + make("subagent-1", 12, {"claude_code_billing": {"cc_prompt_id": "prompt-1", "cc_is_subagent": "true"}}), + make("recap", 20, {"message_id": f"msg-recap-{run_id}"}), + make("main-2", 30, main_loop(2, "prompt-2")), + make("main-2-replay", 35, main_loop(2, "prompt-2b")), + make("subagent-2", 40, {"claude_code_billing": {"cc_prompt_id": "prompt-2b"}}), + ) + ingest( + make("late-title", 32, {"request_hash": f"title-{run_id}"}), + make("late-subagent", 45, {"claude_code_billing": {"cc_prompt_id": "prompt-1"}}), + make("early-probe", 1, {}), + ) + turn = turns(session_id) + require(turn["main-1"] != turn["main-2"], "Claude Code main-loop turns stay distinct") + require(turn["main-2-replay"] == turn["main-2"], "a replay keeps its cc_turn turn") + for name in ["subagent-1", "recap", "late-subagent", "early-probe"]: + require(turn[name] == turn["main-1"], f"{name} joins the first turn") + for name in ["subagent-2", "late-title"]: + require(turn[name] == turn["main-2"], f"{name} joins the second turn") + + other = f"{run_id}-codex-turns" + ingest( + make("codex-1", 10, {"codex_turn_id": f"turn-{run_id}"}, agent="codex", session=other), + make("codex-2", 20, {"response_id": f"response-b-{run_id}"}, agent="codex", session=other), + make("codex-2-response", 21, {"turn_id": f"response-b-{run_id}"}, agent="codex", session=other), + make("unidentified-1", 30, {}, agent="codex", session=other), + make("unidentified-2", 31, {}, agent="codex", session=other), + ) + codex = turns(other) + require(codex["codex-1"] != codex["codex-2"], "other Agents keep provider turn identities") + require(codex["codex-2-response"] != codex["codex-1"], "a turn_id names its own turn") + require( + len({codex["codex-1"], codex["codex-2"], codex["unidentified-1"], codex["unidentified-2"]}) == 4, + "an event without any turn identity is a turn of its own", + ) + print("Claude Code turn attachment API contract passed") diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py index 0d34471..673a8ec 100644 --- a/scripts/tests/session_titles_api.py +++ b/scripts/tests/session_titles_api.py @@ -1,6 +1,5 @@ -"""Session title adapter contract shared by both storage black-box suites.""" +"""Session title contract shared by both storage black-box suites.""" -import base64 import json import urllib.parse import uuid @@ -8,32 +7,23 @@ from blackbox_api import event, request, require, require_status -NAMING_TURN = "01a0f320-138e-7d21-99ea-03a7ae2a9b50" -TASK_TURN = "01a0f320-1350-71c2-9c93-5fc5fd7a6bfb" PROMPT = "Draw a pelican riding a bicycle" CODEX_NAMING = "Generate a concise, single-line task title\nUser prompt:\n" + PROMPT -CLAUDE_NAMING = "You are naming a coding session" def run_title_contract(base_url: str, token: str, run_id: str) -> None: - def make(session, kind, text, *, source="user", agent="codex"): + def make(session, kind, text, *, source="user", agent="codex", metadata=None): value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, {}, run_id) value.pop("attachments", None) value["agent"]["name"] = agent value["device"]["host_name"] = f"title-contract-{run_id}" value["observed_at"] = "2026-09-30T00:00:01Z" if kind == "request" else "2026-09-30T00:00:02Z" - value["metadata"] = {"codex_turn_id": NAMING_TURN if source == "thread_title" else TASK_TURN} + value["metadata"] = dict(metadata or {}) if kind == "request": - payload = { - "input": [{"role": "user", "content": [{"type": "input_text", "text": text}]}], - "client_metadata": {"x-codex-turn-metadata": json.dumps({ - "thread_source": source, - })}, + value["metadata"]["conversation"] = { + "messages": [{"role": "user", "text": text}], + "thread_source": source, "completeness": "complete", } - value["metadata"]["raw_body"] = {"parts": [{ - "truncated": False, - "content_base64": base64.b64encode(json.dumps(payload).encode()).decode(), - }]} return value def ingest(*events): @@ -49,36 +39,40 @@ def session(name): require_status(result, 200, "title outline") return result.json()["session"] - # Response-first and raw-only naming requests exercise both storage paths. - naming = make("naming", "request", CODEX_NAMING, source="thread_title") - naming["text"] = None - ingest(make("naming", "response", '{"title":"Draw a pelican"}', source="thread_title")) - ingest(naming) - ingest(make("task", "request", PROMPT)) - task = session("task") - require(task["title"] == "Draw a pelican", "raw Codex task receives its generated title") - require(session("naming")["parent_session_pk"] == task["session_pk"], "title source links to the task") - - other_device = make("other-device", "request", PROMPT) - other_device["device"]["host_name"] += "-other" - ingest(other_device, make("guardian", "request", PROMPT, source="guardian_review")) - require(session("naming")["parent_session_pk"] == task["session_pk"], "unrelated requests do not withdraw the link") + # Claude Code: the collector reports the naming call and its title, in + # either arrival order relative to the session's first user input. + naming = {"purpose": "session_title"} + for name, title_first in [("claude", False), ("claude-late-input", True)]: + events = [ + make(name, "request", "fix the build", agent="claude-code"), + make(name, "request", "fix the build", agent="claude-code", metadata=naming), + make(name, "response", '{"title":"Build fix"}', agent="claude-code", + metadata={**naming, "session_title": "Build fix"}), + ] + if title_first: + events.reverse() + for value in events: + ingest(value) + value = session(name) + require(value["title"] == "Build fix", f"{name}: the collector title names the session") + require(value["purpose"] is None, f"{name}: a session naming itself is not a title thread") - ingest(make("conflict", "request", PROMPT)) - require(session("naming")["parent_session_pk"] is None, "a real second candidate withdraws the link") - require(session("task")["title"] is None, "ambiguous title is removed from its former task") + # Codex: the naming thread is marked; its task keeps its first user input. + ingest(make("naming", "request", CODEX_NAMING, source="thread_title"), + make("naming", "response", '{"title":"Draw a pelican"}', source="thread_title")) + value = session("naming") + require(value["purpose"] == "title_generation", "Codex naming threads are marked") + require(value["title"] is None and value["parent_session_pk"] is None, "Codex naming threads are not linked") + ingest(make("task", "request", PROMPT)) + require(session("task")["title"] == PROMPT, "Codex tasks keep their first user input") - for agent, name, instruction, expected in [ - ("claude-code", "claude", CLAUDE_NAMING, "Build fix"), - ("cursor", "generic-claude", CLAUDE_NAMING, CLAUDE_NAMING), - ("cursor", "generic-codex", CODEX_NAMING, " ".join(CODEX_NAMING.split())[:79] + "…"), - ("codex", "quoted-user", CODEX_NAMING, None), - ]: - ingest(make(name, "request", instruction, agent=agent), + # Instruction text and title-shaped replies never name a session by themselves. + for agent, name in [("claude-code", "quoted-claude"), ("cursor", "generic-codex")]: + ingest(make(name, "request", CODEX_NAMING, agent=agent), make(name, "response", '{"title":"Build fix"}', agent=agent)) value = session(name) - require(value["title"] == expected, f"{agent}/{name} follows its adapter policy") - require(value["purpose"] is None, f"{agent}/{name} is not an auxiliary title thread") + require(value["title"] == " ".join(CODEX_NAMING.split())[:79] + "…", f"{agent}/{name} keeps its first input") + require(value["purpose"] is None, f"{agent}/{name} is not a title thread") first = make("default", "request", " 修复\n登录问题 " + "好" * 90, agent="cursor") ingest(first) @@ -98,4 +92,4 @@ def session(name): pk = session("default")["session_pk"] listed = next(item for item in result.json()["items"] if item["session_pk"] == pk) require(listed["title"] == "Earlier request", "session list and outline share the default title") - print("session title adapter API contract passed") + print("session title API contract passed") From c45a9d1a9a54b70aa0b6491a41dc143db10c8ebc Mon Sep 17 00:00:00 2001 From: Auceptin Date: Tue, 13 Oct 2026 01:13:18 -0700 Subject: [PATCH 24/30] build: satisfy Clippy 1.99 lints CI installs the latest stable toolchain, now 1.99.0. Its `redundant_field_names` fires on code generated by diesel_derives 2.3.9 for `QueryableByName`; diesel 2.3.14 / diesel_derives 2.3.10 no longer trigger it (lockfile only). Empty-collection test asserts now use `assert_eq!` for the new `assert_is_empty` lint. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 8 ++++---- crates/abyss-backend/src/storage/sqlite/session_list.rs | 2 +- crates/abyss-backend/src/usage/session_list.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38200e1..eba7807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,9 +303,9 @@ checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "diesel" -version = "2.3.12" +version = "2.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715377c6e464cb44bb89bd8487584240516c8d5052bc645d6babc50bb8be46c3" +checksum = "ebe9f7eaef33febd60290c5a9f3b0571d03c4c04a24739cd64808ce30e65a5d3" dependencies = [ "bitflags", "byteorder", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.3.9" +version = "2.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" +checksum = "ecbd51fb6c020672543641167efa4e6417ff7ad76849ed556ace3595e72de03a" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index e2bbbe2..cc9f761 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -321,7 +321,7 @@ mod tests { llm_provider: vec!["anthropic".to_owned()], ..SessionListQuery::default() }; - assert!(list(&mut database, other).is_empty()); + assert_eq!(list(&mut database, other), Vec::new()); } #[test] diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index e7516a6..5a9e1aa 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -147,7 +147,7 @@ mod tests { fn defaults_to_the_first_page_of_every_agent() { let query = SessionListQuery::default().validate().expect("valid"); assert_eq!((query.page, query.page_size, query.offset), (1, 20, 0)); - assert!(query.agent_names.is_empty()); + assert_eq!(query.agent_names, Vec::::new()); } #[test] @@ -184,7 +184,7 @@ mod tests { } .validate() .expect("valid"); - assert!(blank.llm_models.is_empty()); + assert_eq!(blank.llm_models, Vec::::new()); } #[test] From 59a4daa5198744e2a4c7f9909101e6b3e18cb17e Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:33 -0700 Subject: [PATCH 25/30] refactor(schema)!: consolidate migrations into one initial schema The project is not released, so local databases are rebuilt instead of migrated. SQLite and PostgreSQL each keep a single `20261014150000_initial_schema`; the new version makes a database created from the old migrations fail at migration time instead of running against a different schema. - Fold in `session_titles` (with `parent_session_pk`) and the SQLite covering indexes behind the usage summary and the session list. - Add `agent_turns.turn_key` with `UNIQUE (session_pk, turn_key)`, and the exchange identities `stream_id`, `response_id`, and `previous_response_id` on `llm_usage_events`. - Drop `disjoint_token_counters` (it only rewrote old rows), the `schema_backfills` table with the startup title backfill, the unused `session_titles.naming_turn_id` / `naming_prompt` columns, and the SQLite FTS `commands` / `file_paths` columns. The Diesel schemas follow. Session title capture, sharing these files with the backfill, now takes the collector's `naming_thread` flag. Co-Authored-By: Claude Opus 5.5 --- .../down.sql | 2 - .../up.sql | 18 -- .../20260930000000_session_titles/down.sql | 2 - .../20260930000000_session_titles/up.sql | 18 -- .../20261001000000_codex_title_links/down.sql | 3 - .../20261001000000_codex_title_links/up.sql | 6 - .../down.sql | 2 - .../up.sql | 13 -- .../down.sql | 1 + .../up.sql | 45 ++++- .../down.sql | 2 - .../up.sql | 18 -- .../20260930000000_session_titles/down.sql | 2 - .../20260930000000_session_titles/up.sql | 18 -- .../20261001000000_codex_title_links/down.sql | 4 - .../20261001000000_codex_title_links/up.sql | 7 - .../down.sql | 1 + .../up.sql | 33 +++- crates/abyss-backend/src/db/schema.rs | 18 +- crates/abyss-backend/src/storage/postgres.rs | 2 - .../src/storage/postgres/session_titles.rs | 108 ++---------- .../abyss-backend/src/storage/sqlite/mod.rs | 1 - .../src/storage/sqlite/schema.rs | 16 +- .../src/storage/sqlite/session_titles.rs | 166 ++---------------- 24 files changed, 110 insertions(+), 396 deletions(-) delete mode 100644 crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql delete mode 100644 crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql rename crates/abyss-backend/migrations-sqlite/{20260827000000_initial_schema => 20261014150000_initial_schema}/down.sql (91%) rename crates/abyss-backend/migrations-sqlite/{20260827000000_initial_schema => 20261014150000_initial_schema}/up.sql (73%) delete mode 100644 crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql delete mode 100644 crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql delete mode 100644 crates/abyss-backend/migrations/20260930000000_session_titles/down.sql delete mode 100644 crates/abyss-backend/migrations/20260930000000_session_titles/up.sql delete mode 100644 crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql delete mode 100644 crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql rename crates/abyss-backend/migrations/{20260824000000_initial_schema => 20261014150000_initial_schema}/down.sql (92%) rename crates/abyss-backend/migrations/{20260824000000_initial_schema => 20261014150000_initial_schema}/up.sql (84%) diff --git a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql deleted file mode 100644 index 0969841..0000000 --- a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- The original input/total split cannot be recovered after the rewrite. -SELECT 1; diff --git a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql b/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql deleted file mode 100644 index acadb91..0000000 --- a/crates/abyss-backend/migrations-sqlite/20260927000000_disjoint_token_counters/up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Older collectors counted OpenAI-style cached input inside input_tokens and --- set request totals to input_tokens alone, dropping Anthropic cache reads and --- writes. Rewrite stored rows to disjoint input, cache read, cache write, and --- output counters with total_tokens as their sum. --- --- Anthropic Messages already reports cache counters outside input_tokens. --- Provider names cannot identify the protocol because gateway traffic records --- the gateway host, so Anthropic harnesses are excluded by agent name as well, --- and rows whose input is smaller than their cached input are left untouched. -UPDATE llm_usage_events -SET input_tokens = input_tokens - cache_read_tokens - cache_write_tokens -WHERE llm_provider <> 'anthropic' - AND agent_name NOT IN ('claude-code', 'claude-desktop') - AND cache_read_tokens + cache_write_tokens > 0 - AND input_tokens >= cache_read_tokens + cache_write_tokens; - -UPDATE llm_usage_events -SET total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens; diff --git a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql deleted file mode 100644 index 76cec66..0000000 --- a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE schema_backfills; -DROP TABLE session_titles; diff --git a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql b/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql deleted file mode 100644 index ff6ea06..0000000 --- a/crates/abyss-backend/migrations-sqlite/20260930000000_session_titles/up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Session names read back from the Agents' own title-generation calls, kept --- apart from agent_sessions so session rows keep their shape. purpose marks --- sessions that exist only to generate a title, such as Codex naming threads. -CREATE TABLE session_titles ( - session_pk TEXT PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, - user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, - title TEXT, - title_event_pk TEXT REFERENCES llm_usage_events(id) ON DELETE SET NULL, - title_observed_at INTEGER, - purpose TEXT, - updated_at INTEGER NOT NULL -) STRICT; - --- One-time data backfills that run in application code after migrations. -CREATE TABLE schema_backfills ( - name TEXT PRIMARY KEY, - completed_at INTEGER NOT NULL -) STRICT; diff --git a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql deleted file mode 100644 index 9603ac4..0000000 --- a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/down.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE session_titles DROP COLUMN parent_session_pk; -ALTER TABLE session_titles DROP COLUMN naming_prompt; -ALTER TABLE session_titles DROP COLUMN naming_turn_id; diff --git a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql b/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql deleted file mode 100644 index ee84ecd..0000000 --- a/crates/abyss-backend/migrations-sqlite/20261001000000_codex_title_links/up.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Codex names a task from a separate naming thread. naming_turn_id and --- naming_prompt keep what that thread's request shows about the task it names, --- and parent_session_pk records the session it was matched to. -ALTER TABLE session_titles ADD COLUMN naming_turn_id TEXT; -ALTER TABLE session_titles ADD COLUMN naming_prompt TEXT; -ALTER TABLE session_titles ADD COLUMN parent_session_pk TEXT REFERENCES agent_sessions(id) ON DELETE SET NULL; diff --git a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql deleted file mode 100644 index df4680b..0000000 --- a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX llm_usage_events_session_activity_idx; -DROP INDEX llm_usage_events_summary_cover_idx; diff --git a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql b/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql deleted file mode 100644 index f666be5..0000000 --- a/crates/abyss-backend/migrations-sqlite/20261002000000_usage_covering_indexes/up.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Event rows hold whole request bodies in text and metadata, so reading a --- token column from the row walks through megabytes of overflow pages. These --- indexes carry every column the usage summary and the session list read, --- letting both answer from the index alone. -CREATE INDEX llm_usage_events_summary_cover_idx - ON llm_usage_events ( - user_id, observed_at, agent_name, llm_provider, llm_model, event_type, - device_context_id, session_pk, turn_pk, session_id, - input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, - reasoning_tokens, total_tokens - ); -CREATE INDEX llm_usage_events_session_activity_idx - ON llm_usage_events (session_pk, observed_at, total_tokens); diff --git a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql similarity index 91% rename from crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql rename to crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql index 6c08204..fbf1f4a 100644 --- a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql +++ b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql @@ -1,3 +1,4 @@ +DROP TABLE session_titles; DROP TABLE usage_events_fts; DROP TABLE agent_diagnostic_capture_events; DROP TABLE agent_diagnostic_captures; diff --git a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql similarity index 73% rename from crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql rename to crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql index cafdfbf..483d88e 100644 --- a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql +++ b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql @@ -47,16 +47,21 @@ CREATE TABLE agent_sessions ( CREATE INDEX agent_sessions_user_time_idx ON agent_sessions (user_id, started_at DESC); +-- turn_key is the turn the Agent named for its requests, unique within the +-- session; it is NULL for the turn opened by requests naming none. turn_index +-- is the turn's position in the session by start time. CREATE TABLE agent_turns ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, session_pk TEXT NOT NULL REFERENCES agent_sessions(id) ON DELETE CASCADE, turn_index INTEGER NOT NULL CHECK (turn_index >= 1), + turn_key TEXT, started_at INTEGER NOT NULL, ended_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, - UNIQUE (session_pk, turn_index) + UNIQUE (session_pk, turn_index), + UNIQUE (session_pk, turn_key) ) STRICT; CREATE INDEX agent_turns_user_session_idx @@ -85,6 +90,12 @@ CREATE TABLE llm_usage_events ( reasoning_tokens INTEGER NOT NULL DEFAULT 0 CHECK (reasoning_tokens >= 0), total_tokens INTEGER NOT NULL DEFAULT 0 CHECK (total_tokens >= 0), observed_at INTEGER NOT NULL, + -- Agent-declared thread of the exchange; equal values continue one conversation. + stream_id TEXT, + -- Provider identity of the response the exchange produced. + response_id TEXT, + -- Response whose conversation the exchange's request continues. + previous_response_id TEXT, metadata TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata)), created_at INTEGER NOT NULL ) STRICT; @@ -154,7 +165,35 @@ CREATE VIRTUAL TABLE usage_events_fts USING fts5( content, tool_names, tool_content, - commands, - file_paths, tokenize = 'unicode61' ); + + +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads, +-- and parent_session_pk the task session such a thread was matched to. +CREATE TABLE session_titles ( + session_pk TEXT PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title TEXT, + title_event_pk TEXT REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at INTEGER, + purpose TEXT, + parent_session_pk TEXT REFERENCES agent_sessions(id) ON DELETE SET NULL, + updated_at INTEGER NOT NULL +) STRICT; + +-- Event rows hold whole request bodies in text and metadata, so reading a +-- token column from the row walks through megabytes of overflow pages. These +-- indexes carry every column the usage summary and the session list read, +-- letting both answer from the index alone. +CREATE INDEX llm_usage_events_summary_cover_idx + ON llm_usage_events ( + user_id, observed_at, agent_name, llm_provider, llm_model, event_type, + device_context_id, session_pk, turn_pk, session_id, + input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, + reasoning_tokens, total_tokens + ); +CREATE INDEX llm_usage_events_session_activity_idx + ON llm_usage_events (session_pk, observed_at, total_tokens); diff --git a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql deleted file mode 100644 index 0969841..0000000 --- a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- The original input/total split cannot be recovered after the rewrite. -SELECT 1; diff --git a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql b/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql deleted file mode 100644 index acadb91..0000000 --- a/crates/abyss-backend/migrations/20260927000000_disjoint_token_counters/up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Older collectors counted OpenAI-style cached input inside input_tokens and --- set request totals to input_tokens alone, dropping Anthropic cache reads and --- writes. Rewrite stored rows to disjoint input, cache read, cache write, and --- output counters with total_tokens as their sum. --- --- Anthropic Messages already reports cache counters outside input_tokens. --- Provider names cannot identify the protocol because gateway traffic records --- the gateway host, so Anthropic harnesses are excluded by agent name as well, --- and rows whose input is smaller than their cached input are left untouched. -UPDATE llm_usage_events -SET input_tokens = input_tokens - cache_read_tokens - cache_write_tokens -WHERE llm_provider <> 'anthropic' - AND agent_name NOT IN ('claude-code', 'claude-desktop') - AND cache_read_tokens + cache_write_tokens > 0 - AND input_tokens >= cache_read_tokens + cache_write_tokens; - -UPDATE llm_usage_events -SET total_tokens = input_tokens + cache_read_tokens + cache_write_tokens + output_tokens; diff --git a/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql b/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql deleted file mode 100644 index 76cec66..0000000 --- a/crates/abyss-backend/migrations/20260930000000_session_titles/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE schema_backfills; -DROP TABLE session_titles; diff --git a/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql b/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql deleted file mode 100644 index 51d64c6..0000000 --- a/crates/abyss-backend/migrations/20260930000000_session_titles/up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Session names read back from the Agents' own title-generation calls, kept --- apart from agent_sessions so session rows keep their shape. purpose marks --- sessions that exist only to generate a title, such as Codex naming threads. -CREATE TABLE session_titles ( - session_pk uuid PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, - user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, - title text, - title_event_pk uuid REFERENCES llm_usage_events(id) ON DELETE SET NULL, - title_observed_at timestamptz, - purpose text, - updated_at timestamptz NOT NULL DEFAULT now() -); - --- One-time data backfills that run in application code after migrations. -CREATE TABLE schema_backfills ( - name text PRIMARY KEY, - completed_at timestamptz NOT NULL DEFAULT now() -); diff --git a/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql b/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql deleted file mode 100644 index 34b83c0..0000000 --- a/crates/abyss-backend/migrations/20261001000000_codex_title_links/down.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE session_titles - DROP COLUMN parent_session_pk, - DROP COLUMN naming_prompt, - DROP COLUMN naming_turn_id; diff --git a/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql b/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql deleted file mode 100644 index 6e56246..0000000 --- a/crates/abyss-backend/migrations/20261001000000_codex_title_links/up.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Codex names a task from a separate naming thread. naming_turn_id and --- naming_prompt keep what that thread's request shows about the task it names, --- and parent_session_pk records the session it was matched to. -ALTER TABLE session_titles - ADD COLUMN naming_turn_id text, - ADD COLUMN naming_prompt text, - ADD COLUMN parent_session_pk uuid REFERENCES agent_sessions(id) ON DELETE SET NULL; diff --git a/crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql b/crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql similarity index 92% rename from crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql rename to crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql index 1e929fa..06486fa 100644 --- a/crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql +++ b/crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql @@ -1,5 +1,6 @@ -- Drop in reverse dependency order so every foreign key remains valid while -- the consolidated standalone schema is removed. +DROP TABLE IF EXISTS session_titles; DROP TABLE IF EXISTS search_outbox; DROP TABLE IF EXISTS agent_diagnostic_capture_events; DROP TABLE IF EXISTS agent_diagnostic_captures; diff --git a/crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql b/crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql similarity index 84% rename from crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql rename to crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql index 551ee1b..8faca4a 100644 --- a/crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql +++ b/crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql @@ -33,7 +33,7 @@ CREATE TABLE devices ( CREATE INDEX devices_user_seen_idx ON devices (user_id, last_seen_at DESC); --- Sessions use the Agent-native session ID plus canonical Agent name. The +-- Sessions use the Agent-native session ID plus the reported Agent name. The -- device reference follows the most recently observed device for that session. CREATE TABLE agent_sessions ( id uuid PRIMARY KEY, @@ -53,18 +53,21 @@ CREATE TABLE agent_sessions ( CREATE INDEX agent_sessions_user_time_idx ON agent_sessions (user_id, started_at DESC); --- Turn indexes are backend-normalized during ingest when stable provider or --- Agent metadata reveals that a collector restarted its local counter. +-- turn_key is the turn the Agent named for its requests, unique within the +-- session; it is NULL for the turn opened by requests naming none. turn_index +-- is the turn's position in the session by start time. CREATE TABLE agent_turns ( id uuid PRIMARY KEY, user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, session_pk uuid NOT NULL REFERENCES agent_sessions(id) ON DELETE CASCADE, turn_index integer NOT NULL CHECK (turn_index >= 1), + turn_key text, started_at timestamptz NOT NULL, ended_at timestamptz, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), - UNIQUE (session_pk, turn_index) + UNIQUE (session_pk, turn_index), + UNIQUE (session_pk, turn_key) ); CREATE INDEX agent_turns_user_session_idx @@ -95,6 +98,12 @@ CREATE TABLE llm_usage_events ( reasoning_tokens bigint NOT NULL DEFAULT 0 CHECK (reasoning_tokens >= 0), total_tokens bigint NOT NULL DEFAULT 0 CHECK (total_tokens >= 0), observed_at timestamptz NOT NULL, + -- Agent-declared thread of the exchange; equal values continue one conversation. + stream_id text, + -- Provider identity of the response the exchange produced. + response_id text, + -- Response whose conversation the exchange's request continues. + previous_response_id text, metadata jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL DEFAULT now() ); @@ -186,3 +195,19 @@ CREATE TABLE search_outbox ( CREATE INDEX search_outbox_pending_idx ON search_outbox (available_at, id) WHERE processed_at IS NULL AND dead_lettered_at IS NULL; + + +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads, +-- and parent_session_pk the task session such a thread was matched to. +CREATE TABLE session_titles ( + session_pk uuid PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title text, + title_event_pk uuid REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at timestamptz, + purpose text, + parent_session_pk uuid REFERENCES agent_sessions(id) ON DELETE SET NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/crates/abyss-backend/src/db/schema.rs b/crates/abyss-backend/src/db/schema.rs index 1adb464..5cbd236 100644 --- a/crates/abyss-backend/src/db/schema.rs +++ b/crates/abyss-backend/src/db/schema.rs @@ -46,13 +46,14 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::{Integer, Nullable, Timestamptz, Uuid}; + use diesel::sql_types::{Integer, Nullable, Text, Timestamptz, Uuid}; agent_turns (id) { id -> Uuid, user_id -> Uuid, session_pk -> Uuid, turn_index -> Integer, + turn_key -> Nullable, started_at -> Timestamptz, ended_at -> Nullable, created_at -> Timestamptz, @@ -88,6 +89,9 @@ diesel::table! { observed_at -> Timestamptz, metadata -> Jsonb, created_at -> Timestamptz, + stream_id -> Nullable, + response_id -> Nullable, + previous_response_id -> Nullable, } } @@ -163,21 +167,10 @@ diesel::table! { title_observed_at -> Nullable, purpose -> Nullable, updated_at -> Timestamptz, - naming_turn_id -> Nullable, - naming_prompt -> Nullable, parent_session_pk -> Nullable, } } -diesel::table! { - use diesel::sql_types::{Text, Timestamptz}; - - schema_backfills (name) { - name -> Text, - completed_at -> Timestamptz, - } -} - diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -205,7 +198,6 @@ diesel::allow_tables_to_appear_in_same_query!( devices, llm_usage_events, llm_usage_event_attachments, - schema_backfills, search_outbox, session_titles, ); diff --git a/crates/abyss-backend/src/storage/postgres.rs b/crates/abyss-backend/src/storage/postgres.rs index 2f995cf..70b592d 100644 --- a/crates/abyss-backend/src/storage/postgres.rs +++ b/crates/abyss-backend/src/storage/postgres.rs @@ -47,8 +47,6 @@ impl PostgresEsBackend { db::run_migrations(&pool).map_err(|error| { AppError::internal(format!("run PostgreSQL migrations: {error}")) })?; - let mut connection = pool.get()?; - session_titles::backfill(&mut connection)?; } let search = config.search.as_ref().map(SearchService::new).transpose()?; diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs index 77862a4..f9686f0 100644 --- a/crates/abyss-backend/src/storage/postgres/session_titles.rs +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -1,14 +1,12 @@ //! PostgreSQL persistence of session names reported by the collector. use chrono::{DateTime, Utc}; -use diesel::{ - Connection, ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl, -}; +use diesel::{ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl}; use serde_json::Value; use uuid::Uuid; use crate::{ - db::schema::{agent_sessions, llm_usage_events, schema_backfills, session_titles}, + db::schema::{llm_usage_events, session_titles}, error::AppError, usage::{ SessionInfo, @@ -16,9 +14,6 @@ use crate::{ }, }; -/// Names existing sessions using their first user input. -const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; - /// Stored title, purpose, and matched task session of one session. type StoredTitle = (Option, Option, Option); @@ -29,6 +24,7 @@ pub(super) struct CapturedEvent<'a> { pub(super) event_pk: Uuid, pub(super) event_type: &'a str, pub(super) metadata: &'a Value, + pub(super) naming_thread: bool, pub(super) observed_at: DateTime, } @@ -42,14 +38,15 @@ pub(super) fn capture( event: &CapturedEvent<'_>, ) -> Result<(), AppError> { let session = event.session_pk; - let agent_title = match TitleContribution::of(event.event_type, event.metadata) { - None => return Ok(()), - Some(TitleContribution::TitleThread) => { - return mark_title_thread(connection, session, event.user_id); - } - Some(TitleContribution::Agent(title)) => Some(title), - Some(TitleContribution::UserInput) => None, - }; + let agent_title = + match TitleContribution::of(event.event_type, event.metadata, event.naming_thread) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; let stored = session_titles::table .find(session) .select(( @@ -134,87 +131,6 @@ fn mark_title_thread( Ok(()) } -/// Runs the one-time scan that names sessions stored before titles existed. -pub(super) fn backfill(connection: &mut PgConnection) -> Result<(), AppError> { - let done = schema_backfills::table - .find(DEFAULT_BACKFILL_NAME) - .select(schema_backfills::name) - .first::(connection) - .optional()? - .is_some(); - if done { - return Ok(()); - } - connection.transaction(|transaction| { - backfill_defaults(transaction)?; - diesel::insert_into(schema_backfills::table) - .values(( - schema_backfills::name.eq(DEFAULT_BACKFILL_NAME), - schema_backfills::completed_at.eq(Utc::now()), - )) - .execute(transaction)?; - Ok::<_, AppError>(()) - }) -} - -/// Visit requests in time order until the first readable user input is found. -fn backfill_defaults(connection: &mut PgConnection) -> Result<(), AppError> { - let sessions = agent_sessions::table - .select((agent_sessions::id, agent_sessions::user_id)) - .load::<(Uuid, Uuid)>(connection)?; - for (session, user_id) in sessions { - let mut offset = 0_i64; - loop { - let requests = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(session)) - .filter(llm_usage_events::event_type.eq("request")) - .order_by(( - llm_usage_events::observed_at.asc(), - llm_usage_events::id.asc(), - )) - .offset(offset) - .limit(16) - .select(( - llm_usage_events::id, - llm_usage_events::observed_at, - llm_usage_events::metadata, - )) - .load::<(Uuid, DateTime, Value)>(connection)?; - if requests.is_empty() { - break; - } - let mut found = false; - for (event_pk, observed_at, metadata) in requests { - let user_input = matches!( - TitleContribution::of("request", &metadata), - Some(TitleContribution::UserInput) - ); - if !user_input || default_title(&metadata).is_none() { - continue; - } - capture( - connection, - &CapturedEvent { - user_id, - session_pk: session, - event_pk, - event_type: "request", - metadata: &metadata, - observed_at, - }, - )?; - found = true; - break; - } - if found { - break; - } - offset = offset.saturating_add(16); - } - } - Ok(()) -} - /// Adds the stored title, purpose, and matched task session to a session. pub(super) fn fill( connection: &mut PgConnection, diff --git a/crates/abyss-backend/src/storage/sqlite/mod.rs b/crates/abyss-backend/src/storage/sqlite/mod.rs index aa39e70..11f2f31 100644 --- a/crates/abyss-backend/src/storage/sqlite/mod.rs +++ b/crates/abyss-backend/src/storage/sqlite/mod.rs @@ -41,7 +41,6 @@ impl SqliteFtsBackend { connection::configure_database(&mut database)?; if config.run_migrations { migrations::run(&mut database)?; - session_titles::backfill(&mut database)?; } } Ok(Self { pool }) diff --git a/crates/abyss-backend/src/storage/sqlite/schema.rs b/crates/abyss-backend/src/storage/sqlite/schema.rs index 605aa01..45eaec8 100644 --- a/crates/abyss-backend/src/storage/sqlite/schema.rs +++ b/crates/abyss-backend/src/storage/sqlite/schema.rs @@ -53,6 +53,7 @@ diesel::table! { user_id -> Text, session_pk -> Text, turn_index -> Integer, + turn_key -> Nullable, started_at -> BigInt, ended_at -> Nullable, created_at -> BigInt, @@ -88,6 +89,9 @@ diesel::table! { observed_at -> BigInt, metadata -> Text, created_at -> BigInt, + stream_id -> Nullable, + response_id -> Nullable, + previous_response_id -> Nullable, } } @@ -144,21 +148,10 @@ diesel::table! { title_observed_at -> Nullable, purpose -> Nullable, updated_at -> BigInt, - naming_turn_id -> Nullable, - naming_prompt -> Nullable, parent_session_pk -> Nullable, } } -diesel::table! { - use diesel::sql_types::{BigInt, Text}; - - schema_backfills (name) { - name -> Text, - completed_at -> BigInt, - } -} - diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -186,6 +179,5 @@ diesel::allow_tables_to_appear_in_same_query!( devices, llm_usage_events, llm_usage_event_attachments, - schema_backfills, session_titles, ); diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs index c27f56f..86ec7e8 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_titles.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use super::{ models::parse_uuid, - schema::{agent_sessions, llm_usage_events, schema_backfills, session_titles}, + schema::{llm_usage_events, session_titles}, }; use crate::{ error::AppError, @@ -17,9 +17,6 @@ use crate::{ }, }; -/// Names existing sessions using their first user input. -const DEFAULT_BACKFILL_NAME: &str = "first_user_titles_v1"; - /// Stored title, purpose, and matched task session of one session. type StoredTitle = (Option, Option, Option); @@ -30,6 +27,7 @@ pub(super) struct CapturedEvent<'a> { pub(super) event_pk: Uuid, pub(super) event_type: &'a str, pub(super) metadata: &'a Value, + pub(super) naming_thread: bool, pub(super) observed_at: i64, } @@ -43,14 +41,15 @@ pub(super) fn capture( event: &CapturedEvent<'_>, ) -> Result<(), AppError> { let session = event.session_pk.to_string(); - let agent_title = match TitleContribution::of(event.event_type, event.metadata) { - None => return Ok(()), - Some(TitleContribution::TitleThread) => { - return mark_title_thread(connection, &session, event.user_id); - } - Some(TitleContribution::Agent(title)) => Some(title), - Some(TitleContribution::UserInput) => None, - }; + let agent_title = + match TitleContribution::of(event.event_type, event.metadata, event.naming_thread) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, &session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; let stored = session_titles::table .find(&session) .select(( @@ -135,88 +134,6 @@ fn mark_title_thread( Ok(()) } -/// Runs the one-time scan that names sessions stored before titles existed. -pub(super) fn backfill(connection: &mut SqliteConnection) -> Result<(), AppError> { - let done = schema_backfills::table - .find(DEFAULT_BACKFILL_NAME) - .select(schema_backfills::name) - .first::(connection) - .optional()? - .is_some(); - if done { - return Ok(()); - } - connection.immediate_transaction(|transaction| { - backfill_defaults(transaction)?; - diesel::insert_into(schema_backfills::table) - .values(( - schema_backfills::name.eq(DEFAULT_BACKFILL_NAME), - schema_backfills::completed_at.eq(Utc::now().timestamp_micros()), - )) - .execute(transaction)?; - Ok::<_, AppError>(()) - }) -} - -/// Visit requests in time order until the first readable user input is found. -fn backfill_defaults(connection: &mut SqliteConnection) -> Result<(), AppError> { - let sessions = agent_sessions::table - .select((agent_sessions::id, agent_sessions::user_id)) - .load::<(String, String)>(connection)?; - for (session, user_id) in sessions { - let mut offset = 0_i64; - loop { - let requests = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(&session)) - .filter(llm_usage_events::event_type.eq("request")) - .order_by(( - llm_usage_events::observed_at.asc(), - llm_usage_events::id.asc(), - )) - .offset(offset) - .limit(16) - .select(( - llm_usage_events::id, - llm_usage_events::observed_at, - llm_usage_events::metadata, - )) - .load::<(String, i64, String)>(connection)?; - if requests.is_empty() { - break; - } - let mut found = false; - for (event_pk, observed_at, metadata) in requests { - let metadata = serde_json::from_str::(&metadata).unwrap_or(Value::Null); - let user_input = matches!( - TitleContribution::of("request", &metadata), - Some(TitleContribution::UserInput) - ); - if !user_input || default_title(&metadata).is_none() { - continue; - } - capture( - connection, - &CapturedEvent { - user_id: parse_uuid(&user_id, "user id")?, - session_pk: parse_uuid(&session, "session id")?, - event_pk: parse_uuid(&event_pk, "event id")?, - event_type: "request", - metadata: &metadata, - observed_at, - }, - )?; - found = true; - break; - } - if found { - break; - } - offset = offset.saturating_add(16); - } - } - Ok(()) -} - /// Adds the stored title, purpose, and matched task session to a session. pub(super) fn fill( connection: &mut SqliteConnection, @@ -254,12 +171,8 @@ mod tests { use uuid::Uuid; use super::{ - super::{ - migrations, - repository::ingest_events, - schema::{agent_sessions, schema_backfills, session_titles}, - }, - backfill, load, + super::{migrations, repository::ingest_events, schema::agent_sessions}, + load, }; use crate::usage::IngestEventsRequest; @@ -307,7 +220,7 @@ mod tests { "session_id": session, "llm": {"provider": "anthropic", "model": "claude-haiku"}, "event_type": kind, - "token_usage": {"output_tokens": 1_i32} + "token_usage": {"input_tokens": 0_i64, "output_tokens": 1_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64} }); event["metadata"] = metadata; event @@ -486,15 +399,14 @@ mod tests { "2026-09-30T00:00:01Z", "Generate a concise, single-line task title.\nUser prompt:\n画一只鹈鹕", ); - naming["metadata"]["conversation"]["thread_source"] = json!("thread_title"); - let mut task = request( + naming["naming_thread"] = json!(true); + let task = request( "codex", "task", "t1", "2026-09-30T00:00:00Z", "…", ); - task["metadata"]["conversation"]["thread_source"] = json!("user"); ingest( &mut database, &[ @@ -520,50 +432,4 @@ mod tests { (None, Some("title_generation".to_owned())) ); } - - #[test] - fn backfills_default_titles_past_empty_requests_once() { - let mut database = database(); - let mut values: Vec<_> = (0_i32..17_i32) - .map(|index| { - event( - "cursor", - "default", - &format!("empty-{index}"), - "2026-09-30T00:00:00Z", - "request", - json!({}), - ) - }) - .collect(); - values.push(request( - "cursor", - "default", - "first", - "2026-09-30T00:00:01Z", - "First request", - )); - ingest(&mut database, &values); - diesel::delete(session_titles::table) - .execute(&mut database) - .expect("clear titles"); - - backfill(&mut database).expect("backfill"); - assert_eq!( - session_title(&mut database, "default").0.as_deref(), - Some("First request") - ); - diesel::delete(session_titles::table) - .execute(&mut database) - .expect("clear titles"); - backfill(&mut database).expect("second backfill"); - assert_eq!(session_title(&mut database, "default"), (None, None)); - assert_eq!( - schema_backfills::table - .count() - .get_result::(&mut database) - .expect("count"), - 1 - ); - } } From 28a9b1409eeda3b6592b730936c19c16c2ab2a84 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:44 -0700 Subject: [PATCH 26/30] refactor(storage): replace hand-written SQL with the Diesel DSL The PostgreSQL readiness check selects a bound `1` through Diesel. The search outbox backfill becomes `insert_into(..).values(select)` with `on_conflict(..).do_nothing()`, and its completion check `select(not(exists(..)))`; `RawBackfillState` is gone. The same rewrite of the PostgreSQL upserts (`excluded()` and typed `coalesce` / `least` / `greatest`) and of the content-free list placeholder columns lives in the repository files and lands with the ingest change. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/db/mod.rs | 4 +- .../src/storage/postgres/search/outbox.rs | 75 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/crates/abyss-backend/src/db/mod.rs b/crates/abyss-backend/src/db/mod.rs index 0196460..1507008 100644 --- a/crates/abyss-backend/src/db/mod.rs +++ b/crates/abyss-backend/src/db/mod.rs @@ -9,7 +9,7 @@ pub mod models; /// Diesel's compile-time representation of the PostgreSQL schema. pub mod schema; -use diesel::{PgConnection, RunQueryDsl, r2d2::ConnectionManager, sql_query}; +use diesel::{IntoSql, PgConnection, RunQueryDsl, r2d2::ConnectionManager, sql_types::Integer}; use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations}; use crate::{config::Config, error::AppError}; @@ -37,6 +37,6 @@ pub fn run_migrations(pool: &DbPool) -> Result<(), Box Result<(), AppError> { - sql_query("SELECT 1").execute(connection)?; + diesel::select(1_i32.into_sql::()).execute(connection)?; Ok(()) } diff --git a/crates/abyss-backend/src/storage/postgres/search/outbox.rs b/crates/abyss-backend/src/storage/postgres/search/outbox.rs index d7e2876..f9187a7 100644 --- a/crates/abyss-backend/src/storage/postgres/search/outbox.rs +++ b/crates/abyss-backend/src/storage/postgres/search/outbox.rs @@ -10,9 +10,11 @@ use std::collections::HashMap; use chrono::{DateTime, Duration, Utc}; use diesel::{ - ExpressionMethods, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, + ExpressionMethods, IntoSql, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, + SelectableHelper, + dsl::{exists, not}, sql_query, - sql_types::{BigInt, Bool, Integer, Text, Uuid as SqlUuid}, + sql_types::{BigInt, Integer, Text, Uuid as SqlUuid}, }; use uuid::Uuid; @@ -68,42 +70,45 @@ impl SearchOutboxRepository { connection: &mut PgConnection, batch_size: i64, ) -> Result { + let queued_upsert = search_outbox::table + .filter(search_outbox::event_pk.eq(llm_usage_events::id)) + .filter(search_outbox::operation.eq("upsert")) + .select(search_outbox::id); // The unique (event_pk, operation) constraint makes every backfill pass // idempotent and allows startup by multiple replicas. - let queued = sql_query( - "INSERT INTO search_outbox (event_pk, user_id, operation, created_at) \ - SELECT source.id, source.user_id, 'upsert', source.created_at \ - FROM llm_usage_events AS source \ - WHERE NOT EXISTS (\ - SELECT 1 \ - FROM search_outbox AS queued \ - WHERE queued.event_pk = source.id \ - AND queued.operation = 'upsert'\ - ) \ - ORDER BY source.id \ - LIMIT $1 \ - ON CONFLICT (event_pk, operation) DO NOTHING", - ) - .bind::(batch_size) - .execute(connection)?; + let queued = diesel::insert_into(search_outbox::table) + .values( + llm_usage_events::table + .filter(not(exists(queued_upsert))) + .order(llm_usage_events::id) + .limit(batch_size) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + "upsert".into_sql::(), + llm_usage_events::created_at, + )), + ) + .into_columns(( + search_outbox::event_pk, + search_outbox::user_id, + search_outbox::operation, + search_outbox::created_at, + )) + .on_conflict((search_outbox::event_pk, search_outbox::operation)) + .do_nothing() + .execute(connection)?; if i64::try_from(queued).unwrap_or(i64::MAX) >= batch_size { return Ok(false); } - let state = sql_query( - "SELECT NOT EXISTS (\ - SELECT 1 \ - FROM llm_usage_events AS source \ - WHERE NOT EXISTS (\ - SELECT 1 \ - FROM search_outbox AS queued \ - WHERE queued.event_pk = source.id \ - AND queued.operation = 'upsert'\ - )\ - ) AS complete", - ) - .get_result::(connection)?; - Ok(state.complete) + diesel::select(not(exists( + llm_usage_events::table + .filter(not(exists(queued_upsert))) + .select(llm_usage_events::id), + ))) + .get_result::(connection) + .map_err(AppError::from) } /// Leases pending rows and resolves their source events into bulk operations. @@ -313,12 +318,6 @@ struct RawClaimedOutboxTask { attempt_count: i32, } -#[derive(QueryableByName)] -struct RawBackfillState { - #[diesel(sql_type = Bool)] - complete: bool, -} - fn retry_delay(attempt_count: i32) -> i64 { let exponent = u32::try_from(attempt_count.clamp(1_i32, 8_i32)).unwrap_or(8_u32); 2_i64.saturating_pow(exponent).min(MAX_RETRY_DELAY_SECONDS) From 1587b9cd8e4bbb1325bc84a00a074876314260a0 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:44 -0700 Subject: [PATCH 27/30] feat(ingest)!: store AgentEvent values as reported and place turns by turn_id The backend uses the values abyss-rs sends instead of reinterpreting them. - Agent names, providers, devices, session ids, models, and text are stored exactly as received: no trimming, slugs, case folding, Claude alias folding, or `hostname` alias, and empty text stays empty. Claude Desktop is no longer merged into Claude Code. Agent and provider filters match exactly, only trimming and de-duplicating the query values. - Token counters are stored as sent and absent ones as zero; totals are no longer derived and negative counts no longer rejected. - Events are placed by the top-level `turn_id`: events naming a turn share it, the first named turn adopts the turn opened without one, events naming none join the turn in progress when observed, and the session's turns and events are renumbered by start time when a late event reorders them. The metadata turn-identity chain and the Claude Code `cc_prompt_id` attachment are removed. `naming_thread` is passed on to session titles. - `stream_id`, `response_id`, and `previous_response_id` are stored and returned with each event. - Dead reads are removed: the response-chain timeline ordering, the outline `token_usage_estimated`, and the session `working_directory`. - The PostgreSQL upserts use `excluded()` and typed `coalesce` / `least` / `greatest`, and the content-free list binds its placeholder columns. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/db/models.rs | 34 +- crates/abyss-backend/src/search/mod.rs | 62 +- .../src/storage/postgres/repository.rs | 629 ++++++--------- .../src/storage/sqlite/models.rs | 18 +- .../src/storage/sqlite/repository.rs | 753 +++++++++--------- .../src/storage/sqlite/session_list.rs | 30 +- crates/abyss-backend/src/usage/event_order.rs | 474 ++--------- crates/abyss-backend/src/usage/mod.rs | 132 ++- crates/abyss-backend/src/usage/persistence.rs | 295 +------ .../abyss-backend/src/usage/session_list.rs | 29 +- 10 files changed, 879 insertions(+), 1577 deletions(-) diff --git a/crates/abyss-backend/src/db/models.rs b/crates/abyss-backend/src/db/models.rs index 9e3b48a..74cb971 100644 --- a/crates/abyss-backend/src/db/models.rs +++ b/crates/abyss-backend/src/db/models.rs @@ -74,7 +74,7 @@ pub struct AgentSession { pub user_id: Uuid, /// Most recently associated device-context key. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently observed Agent version. pub agent_version: Option, @@ -102,7 +102,7 @@ pub struct NewAgentSession { pub user_id: Uuid, /// Associated device-context key. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Optional Agent version. pub agent_version: Option, @@ -131,8 +131,11 @@ pub struct AgentTurn { pub user_id: Uuid, /// Parent session primary key. pub session_pk: Uuid, - /// Positive turn number unique within the session. + /// Position of the turn in its session by start time. pub turn_index: i32, + /// Turn the Agent named for its requests; `None` for the turn opened by + /// requests naming none. + pub turn_key: Option, /// Earliest event observation in the turn. pub started_at: DateTime, /// Latest event observation in the turn. @@ -153,8 +156,11 @@ pub struct NewAgentTurn { pub user_id: Uuid, /// Parent session primary key. pub session_pk: Uuid, - /// Positive turn number unique within the session. + /// Position of the turn in its session by start time. pub turn_index: i32, + /// Turn the Agent named for its requests; `None` for the turn opened by + /// requests naming none. + pub turn_key: Option, /// Initial earliest observation time. pub started_at: DateTime, /// Initial latest observation time. @@ -182,7 +188,7 @@ pub struct UsageEvent { pub turn_pk: Uuid, /// Collector-generated idempotency key. pub event_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Agent version observed for the event. pub agent_version: Option, @@ -190,7 +196,7 @@ pub struct UsageEvent { pub session_id: String, /// Normalized turn number copied for direct filtering. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -218,6 +224,12 @@ pub struct UsageEvent { pub metadata: Value, /// Server insertion time. pub created_at: DateTime, + /// Agent-declared thread of the exchange. + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + pub previous_response_id: Option, } #[derive(Queryable, Selectable)] @@ -297,7 +309,7 @@ pub struct NewUsageEvent { pub turn_pk: Uuid, /// Collector-generated idempotency key. pub event_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Optional Agent version. pub agent_version: Option, @@ -305,7 +317,7 @@ pub struct NewUsageEvent { pub session_id: String, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -333,6 +345,12 @@ pub struct NewUsageEvent { pub metadata: Value, /// Server insertion time. pub created_at: DateTime, + /// Agent-declared thread of the exchange. + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + pub previous_response_id: Option, } #[derive(Insertable)] diff --git a/crates/abyss-backend/src/search/mod.rs b/crates/abyss-backend/src/search/mod.rs index 718d5fc..fa04921 100644 --- a/crates/abyss-backend/src/search/mod.rs +++ b/crates/abyss-backend/src/search/mod.rs @@ -32,10 +32,10 @@ pub struct SessionSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Optional canonical Agent filter; repeat the key to match any of several. + /// Optional Agent filter, matched exactly; repeat the key to match any of several. #[serde(default)] pub agent_name: Vec, - /// Optional canonical LLM provider filter; repeat the key to match any of several. + /// Optional LLM provider filter, matched exactly; repeat the key to match any of several. #[serde(default)] pub llm_provider: Vec, /// Optional model filter; repeat the key to match any of several. @@ -96,9 +96,9 @@ impl SessionSearchQuery { text, from: self.from, to: self.to, - agent_names: normalized_list(&self.agent_name, "agent_name", canonical_agent_name)?, - llm_providers: normalized_list(&self.llm_provider, "llm_provider", normalize_slug)?, - llm_models: normalized_list(&self.llm_model, "llm_model", str::to_owned)?, + agent_names: normalized_list(&self.agent_name, "agent_name")?, + llm_providers: normalized_list(&self.llm_provider, "llm_provider")?, + llm_models: normalized_list(&self.llm_model, "llm_model")?, event_type, page, page_size, @@ -115,9 +115,9 @@ pub struct ValidatedSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Canonical Agent names to match any of, or empty for every Agent. + /// Trimmed Agent names to match exactly, any of, or empty for every Agent. pub agent_names: Vec, - /// Canonical provider slugs to match any of, or empty for every provider. + /// Trimmed providers to match any of, or empty for every provider. pub llm_providers: Vec, /// Trimmed models to match any of, or empty for every model. pub llm_models: Vec, @@ -155,7 +155,7 @@ pub struct SessionSearchResult { pub session_pk: Uuid, /// Agent-native session identifier. pub session_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently observed Agent version. pub agent_version: Option, @@ -188,7 +188,7 @@ pub struct SessionSearchMatch { pub turn_index: i32, /// Request or response side. pub event_type: String, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -271,43 +271,17 @@ fn normalized_optional(value: Option<&str>, field: &str) -> Result String, -) -> Result, AppError> { +/// Validates and trims every value, then de-duplicates them. +fn normalized_list(values: &[String], field: &str) -> Result, AppError> { let mut normalized = values .iter() - .map(|value| { - normalized_required(value, field, MAX_FILTER_CHARACTERS).map(|value| normalize(&value)) - }) + .map(|value| normalized_required(value, field, MAX_FILTER_CHARACTERS)) .collect::, _>>()?; normalized.sort_unstable(); normalized.dedup(); Ok(normalized) } -fn normalize_slug(value: &str) -> String { - value - .trim() - .to_ascii_lowercase() - .chars() - .map(|character| match character { - '_' | ' ' => '-', - other => other, - }) - .collect() -} - -fn canonical_agent_name(value: &str) -> String { - let normalized = normalize_slug(value); - match normalized.as_str() { - "claude" | "claude-desktop" => "claude-code".to_owned(), - _ => normalized, - } -} - #[cfg(test)] mod tests { use super::{HIGHLIGHT_END, HIGHLIGHT_START, SearchFragment, SessionSearchQuery}; @@ -340,6 +314,18 @@ mod tests { assert!(error.to_string().contains("first 10000 matches")); } + #[test] + fn matches_agent_names_exactly_as_reported() { + let mut agents = query("hello"); + agents.agent_name = vec![ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "claude-code".to_owned(), + ]; + let validated = agents.validate().expect("agent filter is valid"); + assert_eq!(validated.agent_names, ["claude-code", "claude-desktop"]); + } + #[test] fn parses_highlights_without_rendering_html() { let fragment = SearchFragment::parse(&format!( diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index b26fd12..a63169e 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -10,13 +10,16 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - Connection, ExpressionMethods, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, + Connection, ExpressionMethods, IntoSql, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, + SelectableHelper, define_sql_function, dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, sql_types::{ - Array, BigInt, Bool, Bytea, Integer, Jsonb, Nullable, Text, Timestamptz, Uuid as SqlUuid, + Array, BigInt, Bool, Bytea, Integer, Jsonb, Nullable, SingleValue, Text, Timestamptz, + Uuid as SqlUuid, }, + upsert::excluded, }; use uuid::Uuid; @@ -46,21 +49,18 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - SessionTurnAttachment, TurnIdentity, TurnIdentityKind, agent_names_filter, - canonical_agent_name, escape_like_pattern, exact_values_filter, non_empty, - non_negative_count, normalize_limit, normalize_slug, normalize_text, - normalized_optional, normalized_tokens, parse_group_by, providers_filter, - session_turn_attachment, sha256_bytes, turn_identity, unix_epoch, validate_batch, - validate_event_type, + escape_like_pattern, exact_values_filter, non_empty, non_negative_count, + normalize_limit, normalize_text, parse_group_by, sha256_bytes, unix_epoch, + validate_batch, validate_event_type, }, - session_metadata_from_event, }, }; use super::session_titles; -#[cfg(test)] -use crate::usage::persistence::turn_identity_from_metadata; +define_sql_function!(fn coalesce(value: Nullable, fallback: Nullable) -> Nullable); +define_sql_function!(fn least(left: ST, right: ST) -> ST); +define_sql_function!(fn greatest(left: ST, right: ST) -> ST); const SUMMARY_AGGREGATE_SQL: &str = "\ SELECT \ @@ -71,12 +71,7 @@ const SUMMARY_AGGREGATE_SQL: &str = "\ CASE WHEN $3 THEN d.host_name END AS host_name, \ CASE WHEN $3 THEN d.platform END AS platform, \ CASE WHEN $3 THEN d.os_version END AS os_version, \ - CASE \ - WHEN $4 THEN CASE \ - WHEN e.agent_name IN ('claude', 'claude-code', 'claude-desktop') THEN 'claude-code' \ - ELSE e.agent_name \ - END \ - END AS agent_name, \ + CASE WHEN $4 THEN e.agent_name END AS agent_name, \ CASE WHEN $5 THEN e.llm_provider END AS llm_provider, \ CASE WHEN $6 THEN e.llm_model END AS llm_model, \ CASE WHEN $7 THEN e.event_type END AS event_type, \ @@ -397,12 +392,9 @@ type OutlineRow = ( i64, i64, i64, - bool, DateTime, i64, Option, - Option, - Option, ); /// Loads one owner-scoped session timeline without event content bodies. @@ -430,21 +422,12 @@ pub fn session_outline( llm_usage_events::cache_write_tokens, llm_usage_events::reasoning_tokens, llm_usage_events::total_tokens, - sql::("coalesce(metadata -> 'token_usage_estimated' = 'true'::jsonb, false)"), llm_usage_events::observed_at, sql::("coalesce(char_length(text), 0)::bigint"), sql::>(&format!( "CASE WHEN event_type = 'request' THEN right(text, {OUTLINE_PREVIEW_CHARS}) \ ELSE left(text, {OUTLINE_PREVIEW_CHARS}) END" )), - sql::>( - "CASE WHEN jsonb_typeof(metadata -> 'response_id') = 'string' \ - THEN metadata ->> 'response_id' END", - ), - sql::>( - "CASE WHEN jsonb_typeof(metadata -> 'previous_response_id') = 'string' \ - THEN metadata ->> 'previous_response_id' END", - ), )) .load::(connection)?; let event_ids = rows.iter().map(|row| row.0).collect::>(); @@ -471,12 +454,10 @@ pub fn session_outline( cache_write_tokens: row.10, reasoning_tokens: row.11, total_tokens: row.12, - token_usage_estimated: row.13, - observed_at: row.14, - text_chars: row.15, - preview: row.16, + observed_at: row.13, + text_chars: row.14, + preview: row.15, attachment_count: attachment_counts.get(&row.0).copied().unwrap_or(0), - ordering_metadata: UsageEventOutline::ordering_metadata(row.17, row.18), }) .collect::>(); UsageEventTimelineOrder::sort(&mut events); @@ -535,7 +516,7 @@ fn assemble_timeline( session_pk: session.id, user_id: session.user_id, device_context_id: session.device_context_id, - agent_name: canonical_agent_name(&session.agent_name), + agent_name: session.agent_name, agent_version: session.agent_version, session_id: session.session_id, started_at: session.started_at, @@ -686,10 +667,10 @@ fn ingest_one_event( ) -> Result { let observed_at = event.observed_at; let now = Utc::now(); - let agent_name = canonical_agent_name(&event.agent.name); - let agent_version = normalized_optional(event.agent.version.as_deref()); - let provider = normalize_slug(&event.llm.provider); - let model = normalize_text(&event.llm.model); + let agent_name = event.agent.name.clone(); + let agent_version = event.agent.version.clone(); + let provider = event.llm.provider.clone(); + let model = event.llm.model.clone(); let device = upsert_device(connection, user_id, event, observed_at, now)?; let session = upsert_session( @@ -704,17 +685,16 @@ fn ingest_one_event( now, }, )?; - let turn_index = resolve_turn_index(connection, session.id, event, &agent_name, observed_at)?; - let turn = upsert_turn( + let (turn_pk, turn_index) = place_in_turn( connection, user_id, session.id, - turn_index, + event.turn_id.as_deref(), observed_at, now, )?; - let tokens = normalized_tokens(event)?; - let text = normalized_optional(event.text.as_deref()); + let tokens = &event.token_usage; + let text = event.text.clone(); // `event_id` is the idempotency key supplied by the collector. Duplicate // observations are accepted as no-ops instead of failing the whole batch. @@ -725,26 +705,29 @@ fn ingest_one_event( user_id, device_context_id: device.id, session_pk: session.id, - turn_pk: turn.id, - event_id: normalize_text(&event.event_id), + turn_pk, + event_id: event.event_id.clone(), agent_name: agent_name.clone(), agent_version, - session_id: normalize_text(&event.session_id), + session_id: event.session_id.clone(), turn_index, llm_provider: provider, llm_model: model, event_type: event.event_type.as_str().to_owned(), text_sha256: text.as_deref().map(sha256_bytes), text, - input_tokens: tokens.input, - output_tokens: tokens.output, - cache_read_tokens: tokens.cache_read, - cache_write_tokens: tokens.cache_write, - reasoning_tokens: tokens.reasoning, - total_tokens: tokens.total, + input_tokens: tokens.input_tokens.unwrap_or_default(), + output_tokens: tokens.output_tokens.unwrap_or_default(), + cache_read_tokens: tokens.cache_read_tokens.unwrap_or_default(), + cache_write_tokens: tokens.cache_write_tokens.unwrap_or_default(), + reasoning_tokens: tokens.reasoning_tokens.unwrap_or_default(), + total_tokens: tokens.total_tokens.unwrap_or_default(), observed_at, metadata: event.metadata.clone(), created_at: now, + stream_id: event.stream_id.clone(), + response_id: event.response_id.clone(), + previous_response_id: event.previous_response_id.clone(), }) .on_conflict(llm_usage_events::event_id) .do_nothing() @@ -774,6 +757,7 @@ fn ingest_one_event( event_pk: usage_event_id, event_type: event.event_type.as_str(), metadata: &event.metadata, + naming_thread: event.naming_thread, observed_at, }, )?; @@ -823,9 +807,9 @@ fn upsert_device( .values(NewDevice { id: Uuid::now_v7(), user_id, - host_name: normalize_text(&event.device.host_name), - platform: normalize_slug(&event.device.platform), - os_version: normalized_optional(event.device.os_version.as_deref()), + host_name: event.device.host_name.clone(), + platform: event.device.platform.clone(), + os_version: event.device.os_version.clone(), first_seen_at: observed_at, last_seen_at: observed_at, created_at: now, @@ -834,14 +818,14 @@ fn upsert_device( .on_conflict((devices::user_id, devices::host_name, devices::platform)) .do_update() .set(( - devices::os_version.eq(sql::>( - "COALESCE(EXCLUDED.os_version, devices.os_version)", + devices::os_version.eq(coalesce(excluded(devices::os_version), devices::os_version)), + devices::first_seen_at.eq(least( + devices::first_seen_at, + excluded(devices::first_seen_at), )), - devices::first_seen_at.eq(sql::( - "LEAST(devices.first_seen_at, EXCLUDED.first_seen_at)", - )), - devices::last_seen_at.eq(sql::( - "GREATEST(devices.last_seen_at, EXCLUDED.last_seen_at)", + devices::last_seen_at.eq(greatest( + devices::last_seen_at, + excluded(devices::last_seen_at), )), devices::updated_at.eq(now), )) @@ -853,7 +837,7 @@ fn upsert_session( connection: &mut PgConnection, input: UpsertSessionInput<'_>, ) -> Result { - let session_id = normalize_text(&input.event.session_id); + let session_id = input.event.session_id.clone(); diesel::insert_into(agent_sessions::table) .values(NewAgentSession { id: Uuid::now_v7(), @@ -864,7 +848,7 @@ fn upsert_session( session_id, started_at: input.observed_at, ended_at: Some(input.observed_at), - metadata: session_metadata_from_event(&input.event.metadata), + metadata: serde_json::Value::Object(serde_json::Map::new()), created_at: input.now, updated_at: input.now, }) @@ -875,217 +859,180 @@ fn upsert_session( )) .do_update() .set(( - agent_sessions::device_context_id - .eq(sql::("EXCLUDED.device_context_id")), - agent_sessions::agent_version.eq(sql::>( - "COALESCE(EXCLUDED.agent_version, agent_sessions.agent_version)", + agent_sessions::device_context_id.eq(excluded(agent_sessions::device_context_id)), + agent_sessions::agent_version.eq(coalesce( + excluded(agent_sessions::agent_version), + agent_sessions::agent_version, )), - agent_sessions::started_at.eq(sql::( - "LEAST(agent_sessions.started_at, EXCLUDED.started_at)", + agent_sessions::started_at.eq(least( + agent_sessions::started_at, + excluded(agent_sessions::started_at), )), - agent_sessions::ended_at.eq(sql::>( - "GREATEST(COALESCE(agent_sessions.ended_at, EXCLUDED.ended_at), EXCLUDED.ended_at)", + agent_sessions::ended_at.eq(greatest( + coalesce(agent_sessions::ended_at, excluded(agent_sessions::ended_at)), + excluded(agent_sessions::ended_at), )), - agent_sessions::metadata - .eq(sql::("agent_sessions.metadata || EXCLUDED.metadata")), agent_sessions::updated_at.eq(input.now), )) .get_result::(connection) .map_err(AppError::from) } -fn resolve_turn_index( +/// Places an event in a turn of its session and returns the turn's id and +/// position. +/// +/// Events naming the same turn share it, and an event naming none joins the +/// turn in progress when it was observed. Turns are numbered by when they +/// started, so a turn that arrives late moves the turns after it. +fn place_in_turn( connection: &mut PgConnection, + user_id: Uuid, session_pk: Uuid, - event: &IngestUsageEvent, - agent_name: &str, + turn_id: Option<&str>, observed_at: DateTime, -) -> Result { - if let Some(attachment) = session_turn_attachment(agent_name, &event.metadata) - && let Some(turn_index) = - attached_turn_index(connection, session_pk, &attachment, observed_at)? - { - return Ok(turn_index); - } - // Events of one turn share its identity; an event without any identity - // is a turn of its own. - if let Some(identity) = turn_identity(event) - && let Some(existing) = existing_turn_index_for_identity(connection, session_pk, &identity)? - { - return Ok(existing); - } - next_session_turn_index(connection, session_pk) + now: DateTime, +) -> Result<(Uuid, i32), AppError> { + let found = match turn_id { + Some(key) => named_turn(connection, session_pk, key)?, + None => turn_in_progress(connection, session_pk, observed_at)?, + }; + let (turn_pk, turn_index, reordered) = if let Some(turn) = found { + diesel::update(agent_turns::table.find(turn.id)) + .set(( + agent_turns::started_at.eq(turn.started_at.min(observed_at)), + agent_turns::ended_at + .eq(Some(turn.ended_at.unwrap_or(observed_at).max(observed_at))), + agent_turns::updated_at.eq(now), + )) + .execute(connection)?; + (turn.id, turn.turn_index, observed_at < turn.started_at) + } else { + let last = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .select(max(agent_turns::turn_index)) + .get_result::>(connection)?; + let id = Uuid::now_v7(); + let turn_index = last.unwrap_or(0_i32).saturating_add(1_i32); + diesel::insert_into(agent_turns::table) + .values(NewAgentTurn { + id, + user_id, + session_pk, + turn_index, + turn_key: turn_id.map(str::to_owned), + started_at: observed_at, + ended_at: Some(observed_at), + created_at: now, + updated_at: now, + }) + .execute(connection)?; + (id, turn_index, last.is_some()) + }; + let turn_index = if reordered && number_turns(connection, session_pk)? { + agent_turns::table + .find(turn_pk) + .select(agent_turns::turn_index) + .get_result::(connection)? + } else { + turn_index + }; + Ok((turn_pk, turn_index)) } -/// Finds the stored turn a Claude Code side or subagent request belongs to; -/// `None` only when the session has no turn yet. -fn attached_turn_index( +/// The turn named `key` in a session. The first named turn adopts the turn +/// that events naming none opened at the start of the session. +fn named_turn( connection: &mut PgConnection, session_pk: Uuid, - attachment: &SessionTurnAttachment, - observed_at: DateTime, -) -> Result, AppError> { - if let Some(prompt_id) = &attachment.prompt_id { - // Only main-loop events anchor a prompt, and each prompt id belongs to - // one turn. Walking the session index from its latest turn usually - // stops within the current turn instead of detoasting every megabyte - // metadata row of the session. - let prompt_turn = sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 \ - AND metadata->'claude_code_billing'->>'cc_prompt_id' = $2 \ - AND metadata->>'claude_turn_id' IS NOT NULL \ - ORDER BY turn_index DESC, observed_at DESC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(prompt_id) - .get_result::(connection) - .optional()?; - if let Some(row) = prompt_turn { - return Ok(Some(row.turn_index)); - } - } - // Events may arrive out of order, so the observation time picks the turn. - let preceding = agent_turns::table + key: &str, +) -> Result, AppError> { + let named = agent_turns::table .filter(agent_turns::session_pk.eq(session_pk)) - .filter(agent_turns::started_at.le(observed_at)) - .order_by(( - agent_turns::started_at.desc(), - agent_turns::turn_index.desc(), - )) - .select(agent_turns::turn_index) - .first::(connection) + .filter(agent_turns::turn_key.eq(key)) + .select(AgentTurn::as_select()) + .first::(connection) .optional()?; - if preceding.is_some() { - return Ok(preceding); + if named.is_some() { + return Ok(named); } - agent_turns::table + let mut turns = agent_turns::table .filter(agent_turns::session_pk.eq(session_pk)) - .order_by((agent_turns::started_at.asc(), agent_turns::turn_index.asc())) - .select(agent_turns::turn_index) - .first::(connection) - .optional() - .map_err(AppError::from) -} - -fn existing_turn_index_for_identity( - connection: &mut PgConnection, - session_pk: Uuid, - identity: &TurnIdentity, -) -> Result, AppError> { - let row = match &identity.kind { - TurnIdentityKind::CodexTurnId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'codex_turn_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::ClaudeTurnId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'claude_turn_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::TurnId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'turn_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::ResponseId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'response_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::MessageId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'message_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::RequestHash => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'request_hash' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), + .select(AgentTurn::as_select()) + .limit(2_i64) + .load::(connection)?; + let opening = match turns.as_slice() { + [turn] if turn.turn_key.is_none() => turns.pop(), + _ => None, }; - row.optional() - .map(|row| row.map(|row| row.turn_index)) - .map_err(AppError::from) + if let Some(turn) = &opening { + diesel::update(agent_turns::table.find(turn.id)) + .set(agent_turns::turn_key.eq(key)) + .execute(connection)?; + } + Ok(opening) } -fn next_session_turn_index( +/// The turn in progress when an event naming no turn was observed: the latest +/// turn started by then, else the first turn of the session. +fn turn_in_progress( connection: &mut PgConnection, session_pk: Uuid, -) -> Result { + observed_at: DateTime, +) -> Result, AppError> { + let started = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .filter(agent_turns::started_at.le(observed_at)) + .order_by((agent_turns::started_at.desc(), agent_turns::id.desc())) + .select(AgentTurn::as_select()) + .first::(connection) + .optional()?; + if started.is_some() { + return Ok(started); + } agent_turns::table .filter(agent_turns::session_pk.eq(session_pk)) - .select(max(agent_turns::turn_index)) - .get_result::>(connection) - .map(|value| value.unwrap_or(0_i32).saturating_add(1_i32)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select(AgentTurn::as_select()) + .first::(connection) + .optional() .map_err(AppError::from) } -fn upsert_turn( - connection: &mut PgConnection, - user_id: Uuid, - session_pk: Uuid, - turn_index: i32, - observed_at: DateTime, - now: DateTime, -) -> Result { - diesel::insert_into(agent_turns::table) - .values(NewAgentTurn { - id: Uuid::now_v7(), - user_id, - session_pk, - turn_index, - started_at: observed_at, - ended_at: Some(observed_at), - created_at: now, - updated_at: now, - }) - .on_conflict((agent_turns::session_pk, agent_turns::turn_index)) - .do_update() - .set(( - agent_turns::started_at.eq(sql::( - "LEAST(agent_turns.started_at, EXCLUDED.started_at)", - )), - agent_turns::ended_at.eq(sql::>( - "GREATEST(COALESCE(agent_turns.ended_at, EXCLUDED.ended_at), EXCLUDED.ended_at)", - )), - agent_turns::updated_at.eq(now), - )) - .get_result::(connection) - .map_err(AppError::from) +/// Numbers the turns of a session, and their events, by when the turns +/// started. Writes nothing and returns `false` while the numbers already +/// follow that order. +fn number_turns(connection: &mut PgConnection, session_pk: Uuid) -> Result { + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select((agent_turns::id, agent_turns::turn_index)) + .load::<(Uuid, i32)>(connection)?; + let moved = turns + .iter() + .zip(1_i32..) + .filter(|((_, index), position)| index != position) + .map(|((id, _), position)| (*id, position)) + .collect::>(); + if moved.is_empty() { + return Ok(false); + } + // Moved turns first step above every current number, so no single-row + // update collides with another turn's number or goes below one. + let above = turns.iter().map(|(_, index)| *index).max().unwrap_or(0_i32); + for (id, position) in &moved { + diesel::update(agent_turns::table.find(*id)) + .set(agent_turns::turn_index.eq(above.saturating_add(*position))) + .execute(connection)?; + } + for (id, position) in moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(position)) + .execute(connection)?; + diesel::update(llm_usage_events::table.filter(llm_usage_events::turn_pk.eq(id))) + .set(llm_usage_events::turn_index.eq(position)) + .execute(connection)?; + } + Ok(true) } /// Loads matching events newest first. Without content, text and metadata are @@ -1111,7 +1058,7 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(to)); } - let agent_names = agent_names_filter(filters.agent_names); + let agent_names = exact_values_filter(filters.agent_names); if !agent_names.is_empty() { query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } @@ -1125,7 +1072,7 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - let providers = providers_filter(filters.llm_providers); + let providers = exact_values_filter(filters.llm_providers); if !providers.is_empty() { query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } @@ -1159,8 +1106,8 @@ fn load_filtered_events( llm_usage_events::llm_provider, llm_usage_events::llm_model, llm_usage_events::event_type, - sql::>("NULL"), - sql::>("NULL"), + None::.into_sql::>(), + None::>.into_sql::>(), llm_usage_events::input_tokens, llm_usage_events::output_tokens, llm_usage_events::cache_read_tokens, @@ -1168,8 +1115,11 @@ fn load_filtered_events( llm_usage_events::reasoning_tokens, llm_usage_events::total_tokens, llm_usage_events::observed_at, - sql::("'{}'::jsonb"), + serde_json::json!({}).into_sql::(), llm_usage_events::created_at, + llm_usage_events::stream_id, + llm_usage_events::response_id, + llm_usage_events::previous_response_id, )) .load::(connection) .map_err(AppError::from) @@ -1252,7 +1202,7 @@ fn event_response( platform: device.map(|value| value.platform.clone()), session_pk: event.session_pk, turn_pk: event.turn_pk, - agent_name: canonical_agent_name(&event.agent_name), + agent_name: event.agent_name, agent_version: event.agent_version, session_id: event.session_id, turn_index: event.turn_index, @@ -1268,6 +1218,9 @@ fn event_response( reasoning_tokens: event.reasoning_tokens, total_tokens: event.total_tokens, observed_at: event.observed_at, + stream_id: event.stream_id, + response_id: event.response_id, + previous_response_id: event.previous_response_id, metadata: event.metadata, attachments, } @@ -1397,9 +1350,9 @@ impl SummarySqlFilters { let user_filter_id = user_filter.and_then(|value| Uuid::parse_str(value).ok()); let user_filter_pattern = user_filter.map(|value| format!("%{}%", escape_like_pattern(value))); - let agent_names = agent_names_filter(filters.agent_names); + let agent_names = exact_values_filter(filters.agent_names); let session_ids = exact_values_filter(filters.session_ids); - let providers = providers_filter(filters.llm_providers); + let providers = exact_values_filter(filters.llm_providers); let models = exact_values_filter(filters.llm_models); let event_type = filters.event_type.map(validate_event_type).transpose()?; @@ -1494,12 +1447,6 @@ impl From for SummaryRow { } } -#[derive(QueryableByName)] -struct TurnIndexRow { - #[diesel(sql_type = Integer)] - turn_index: i32, -} - struct UpsertSessionInput<'a> { device: &'a Device, user_id: Uuid, @@ -1515,10 +1462,8 @@ mod tests { use std::collections::{HashMap, HashSet}; use super::{ - EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, TurnIdentityKind, - canonical_agent_name, event_response, latest_optional, normalize_limit, normalize_slug, - normalized_tokens, parse_group_by, turn_identity_from_metadata, validate_batch, - validate_event_type, + EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, event_response, + latest_optional, normalize_limit, parse_group_by, validate_batch, validate_event_type, }; use chrono::{DateTime, TimeZone, Utc}; use serde_json::json; @@ -1529,7 +1474,7 @@ mod tests { error::AppError, usage::{ AgentPayload, DevicePayload, IngestEventsRequest, IngestUsageEvent, LlmPayload, - SummaryRow, TokenUsagePayload, UsageEventType, persistence::agent_name_filter_values, + SummaryRow, TokenUsagePayload, UsageEventType, }, }; @@ -1674,43 +1619,13 @@ mod tests { os_version: includes("device") .then(|| device.and_then(|value| value.os_version.clone())) .flatten(), - agent_name: includes("agent").then(|| canonical_agent_name(&event.agent_name)), + agent_name: includes("agent").then(|| event.agent_name.clone()), llm_provider: includes("provider").then(|| event.llm_provider.clone()), llm_model: includes("model").then(|| event.llm_model.clone()), event_type: includes("event_type").then(|| event.event_type.clone()), } } - #[test] - fn normalize_slug_converts_agent_names() { - assert_eq!( - normalize_slug("Claude Code"), - "claude-code", - "agent names should be normalized to lowercase kebab-case" - ); - } - - #[test] - fn canonical_agent_name_groups_claude_clients() { - assert_eq!(canonical_agent_name("Claude"), "claude-code"); - assert_eq!(canonical_agent_name("Claude Code"), "claude-code"); - assert_eq!(canonical_agent_name("claude-desktop"), "claude-code"); - assert_eq!(canonical_agent_name("Codex"), "codex"); - } - - #[test] - fn agent_name_filter_values_include_historical_claude_slugs() { - assert_eq!( - agent_name_filter_values("Claude Code"), - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] - ); - assert_eq!(agent_name_filter_values("Codex"), vec!["codex".to_owned()]); - } - #[test] fn latest_optional_keeps_the_newer_timestamp() { let early = Utc @@ -1762,13 +1677,13 @@ mod tests { } #[test] - fn summary_sql_filters_normalize_runtime_filters() { + fn summary_sql_filters_trim_and_dedupe_runtime_filters() { let filters = EventFilters { user_id: fixed_uuid(42_u128), from: Some(timestamp(2_u32, 0_u32, 0_u32)), to: None, user_filter: Some(" Alice_% "), - agent_names: &["Claude Desktop".to_owned(), "codex".to_owned()], + agent_names: &[" claude-desktop ".to_owned(), "codex".to_owned()], session_ids: &[" session-1 ".to_owned(), "session-2".to_owned()], session_pk: None, turn_index: None, @@ -1794,14 +1709,11 @@ mod tests { sql_filters.user_filter_pattern.as_deref(), Some("%Alice\\_\\%%") ); - assert_eq!( - sql_filters.agent_names, - ["claude", "claude-code", "claude-desktop", "codex"] - ); + assert_eq!(sql_filters.agent_names, ["claude-desktop", "codex"]); assert_eq!(sql_filters.session_ids, ["session-1", "session-2"]); assert_eq!(sql_filters.session_pk, None); assert_eq!(sql_filters.turn_index, None); - assert_eq!(sql_filters.providers, ["openai"]); + assert_eq!(sql_filters.providers, ["OpenAI"]); assert_eq!(sql_filters.models, ["gpt-5.5", "o3"]); assert_eq!(sql_filters.event_type.as_deref(), Some("response")); } @@ -1821,42 +1733,6 @@ mod tests { ); } - #[test] - fn normalized_tokens_derives_total_from_components() { - let mut event = sample_event(); - event.token_usage = TokenUsagePayload { - input_tokens: Some(11_i64), - output_tokens: Some(13_i64), - cache_read_tokens: Some(2_i64), - cache_write_tokens: Some(3_i64), - reasoning_tokens: Some(5_i64), - total_tokens: None, - }; - - let tokens = normalized_tokens(&event).expect("token usage should normalize"); - assert_eq!( - tokens.total, 29_i64, - "missing total_tokens should sum disjoint counters without adding reasoning twice" - ); - } - - #[test] - fn normalized_tokens_rejects_negative_components() { - let mut event = sample_event(); - event.token_usage = TokenUsagePayload { - input_tokens: Some(-1_i64), - ..TokenUsagePayload::default() - }; - - let Err(error) = normalized_tokens(&event) else { - panic!("negative token usage should be invalid"); - }; - assert!( - matches!(&error, AppError::Validation(message) if message.contains("input_tokens")), - "negative input_tokens should return a validation error" - ); - } - #[test] fn validate_event_type_normalizes_known_values() { assert_eq!( @@ -1881,38 +1757,6 @@ mod tests { ); } - #[test] - fn turn_identity_prefers_codex_turn_id_before_provider_response_id() { - let identity = turn_identity_from_metadata(&json!({ - "codex_turn_id": "turn-a", - "response_id": "resp-a", - "request_hash": "hash-a" - })) - .expect("codex_turn_id should produce a turn identity"); - - assert!( - matches!(identity.kind, TurnIdentityKind::CodexTurnId), - "one Codex turn can span multiple provider response ids" - ); - assert_eq!(identity.value, "turn-a"); - } - - #[test] - fn turn_identity_uses_claude_turn_id_before_provider_message_id() { - let identity = turn_identity_from_metadata(&json!({ - "claude_turn_id": "claude-turn-a", - "message_id": "msg-a", - "request_hash": "hash-a" - })) - .expect("claude_turn_id should produce a turn identity"); - - assert!( - matches!(identity.kind, TurnIdentityKind::ClaudeTurnId), - "one Claude user turn can span multiple provider message ids" - ); - assert_eq!(identity.value, "claude-turn-a"); - } - #[test] fn normalize_limit_uses_fallback_and_clamps_bounds() { assert_eq!( @@ -2003,7 +1847,7 @@ mod tests { } #[test] - fn aggregate_summary_groups_historical_claude_agent_slugs() { + fn aggregate_summary_keeps_claude_desktop_apart_from_claude_code() { let device_context_id = fixed_uuid(1_u128); let mut desktop_event = usage_event( "evt-claude-desktop-request", @@ -2037,16 +1881,25 @@ mod tests { &group_by, ); + let rows = rows + .iter() + .map(|row| { + ( + row.agent_name.as_deref(), + row.requests, + row.responses, + row.total_tokens, + ) + }) + .collect::>(); assert_eq!( - rows.len(), - 1_usize, - "Claude CLI and Desktop rows should aggregate as one product" + rows, + [ + (Some("claude-code"), 0_i64, 1_i64, 12_i64), + (Some("claude-desktop"), 1_i64, 0_i64, 8_i64), + ], + "Claude Desktop is its own Agent, not an alias of Claude Code" ); - let row = rows.first().expect("summary should contain one row"); - assert_eq!(row.agent_name.as_deref(), Some("claude-code")); - assert_eq!(row.requests, 1_i64); - assert_eq!(row.responses, 1_i64); - assert_eq!(row.total_tokens, 20_i64); } #[test] @@ -2097,6 +1950,11 @@ mod tests { version: Some("1.0.0".to_owned()), }, session_id: "session-001".to_owned(), + turn_id: None, + naming_thread: false, + stream_id: None, + response_id: None, + previous_response_id: None, llm: LlmPayload { provider: "OpenAI".to_owned(), model: "gpt-5.5".to_owned(), @@ -2106,10 +1964,10 @@ mod tests { token_usage: TokenUsagePayload { input_tokens: Some(5_i64), output_tokens: Some(7_i64), - cache_read_tokens: None, - cache_write_tokens: None, - reasoning_tokens: None, - total_tokens: None, + cache_read_tokens: Some(0_i64), + cache_write_tokens: Some(0_i64), + reasoning_tokens: Some(0_i64), + total_tokens: Some(12_i64), }, metadata: json!({}), attachments: Vec::new(), @@ -2149,6 +2007,9 @@ mod tests { observed_at: timestamp(1_u32, 0_u32, 0_u32), metadata: json!({}), created_at: timestamp(1_u32, 0_u32, 1_u32), + stream_id: None, + response_id: None, + previous_response_id: None, } } diff --git a/crates/abyss-backend/src/storage/sqlite/models.rs b/crates/abyss-backend/src/storage/sqlite/models.rs index e55bbe8..0bc692f 100644 --- a/crates/abyss-backend/src/storage/sqlite/models.rs +++ b/crates/abyss-backend/src/storage/sqlite/models.rs @@ -82,6 +82,7 @@ pub(super) struct TurnRow { pub(super) user_id: String, pub(super) session_pk: String, pub(super) turn_index: i32, + pub(super) turn_key: Option, pub(super) started_at: i64, pub(super) ended_at: Option, pub(super) created_at: i64, @@ -95,6 +96,7 @@ pub(super) struct NewTurn { pub(super) user_id: String, pub(super) session_pk: String, pub(super) turn_index: i32, + pub(super) turn_key: Option, pub(super) started_at: i64, pub(super) ended_at: Option, pub(super) created_at: i64, @@ -129,6 +131,9 @@ pub(super) struct EventRow { pub(super) observed_at: i64, pub(super) metadata: String, pub(super) created_at: i64, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } #[derive(Insertable)] @@ -158,6 +163,9 @@ pub(super) struct NewEvent { pub(super) observed_at: i64, pub(super) metadata: String, pub(super) created_at: i64, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } #[derive(Queryable, Selectable)] @@ -261,6 +269,9 @@ pub(super) struct EventRecord { pub(super) total_tokens: i64, pub(super) observed_at: DateTime, pub(super) metadata: Value, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } impl TimelineEvent for EventRecord { @@ -268,10 +279,6 @@ impl TimelineEvent for EventRecord { self.turn_index } - fn metadata(&self) -> &Value { - &self.metadata - } - fn observed_at(&self) -> DateTime { self.observed_at } @@ -372,6 +379,9 @@ impl EventRow { total_tokens: self.total_tokens, observed_at: timestamp_from_micros(self.observed_at)?, metadata: parse_json(&self.metadata, "event metadata")?, + stream_id: self.stream_id, + response_id: self.response_id, + previous_response_id: self.previous_response_id, }) } } diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index e2cc433..33690c4 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -4,7 +4,8 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - ExpressionMethods, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, SqliteConnection, + ExpressionMethods, IntoSql, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, + SqliteConnection, dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, @@ -27,13 +28,9 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - SessionTurnAttachment, TurnIdentity, agent_names_filter, canonical_agent_name, - exact_values_filter, non_empty, normalize_limit, normalize_slug, normalize_text, - normalized_optional, normalized_tokens, parse_group_by, providers_filter, - session_turn_attachment, sha256_bytes, turn_identity, validate_batch, - validate_event_type, + exact_values_filter, non_empty, normalize_limit, normalize_text, parse_group_by, + sha256_bytes, validate_batch, validate_event_type, }, - session_metadata_from_event, }, }; @@ -98,7 +95,7 @@ fn ingest_one_event( event: &IngestUsageEvent, attachments: Vec, ) -> Result { - let event_id = normalize_text(&event.event_id); + let event_id = event.event_id.clone(); let exists = llm_usage_events::table .filter(llm_usage_events::event_id.eq(&event_id)) .select(llm_usage_events::id) @@ -111,10 +108,10 @@ fn ingest_one_event( let observed_at = timestamp_to_micros(event.observed_at); let now = timestamp_to_micros(Utc::now()); - let agent_name = canonical_agent_name(&event.agent.name); - let agent_version = normalized_optional(event.agent.version.as_deref()); - let provider = normalize_slug(&event.llm.provider); - let model = normalize_text(&event.llm.model); + let agent_name = event.agent.name.clone(); + let agent_version = event.agent.version.clone(); + let provider = event.llm.provider.clone(); + let model = event.llm.model.clone(); let device_id = upsert_device(connection, user_id, event, observed_at, now)?; let session_pk = upsert_session( connection, @@ -126,17 +123,16 @@ fn ingest_one_event( observed_at, now, )?; - let turn_index = resolve_turn_index(connection, session_pk, event, &agent_name, observed_at)?; - let turn_pk = upsert_turn( + let (turn_pk, turn_index) = place_in_turn( connection, user_id, session_pk, - turn_index, + event.turn_id.as_deref(), observed_at, now, )?; - let tokens = normalized_tokens(event)?; - let text = normalized_optional(event.text.as_deref()); + let tokens = &event.token_usage; + let text = event.text.clone(); let event_pk = Uuid::now_v7(); let metadata = serialize_json(&event.metadata, "event metadata")?; @@ -150,22 +146,25 @@ fn ingest_one_event( event_id, agent_name: agent_name.clone(), agent_version, - session_id: normalize_text(&event.session_id), + session_id: event.session_id.clone(), turn_index, llm_provider: provider, llm_model: model, event_type: event.event_type.as_str().to_owned(), text: text.clone(), text_sha256: text.as_deref().map(sha256_bytes), - input_tokens: tokens.input, - output_tokens: tokens.output, - cache_read_tokens: tokens.cache_read, - cache_write_tokens: tokens.cache_write, - reasoning_tokens: tokens.reasoning, - total_tokens: tokens.total, + input_tokens: tokens.input_tokens.unwrap_or_default(), + output_tokens: tokens.output_tokens.unwrap_or_default(), + cache_read_tokens: tokens.cache_read_tokens.unwrap_or_default(), + cache_write_tokens: tokens.cache_write_tokens.unwrap_or_default(), + reasoning_tokens: tokens.reasoning_tokens.unwrap_or_default(), + total_tokens: tokens.total_tokens.unwrap_or_default(), observed_at, metadata, created_at: now, + stream_id: event.stream_id.clone(), + response_id: event.response_id.clone(), + previous_response_id: event.previous_response_id.clone(), }) .on_conflict(llm_usage_events::event_id) .do_nothing() @@ -184,6 +183,7 @@ fn ingest_one_event( event_pk, event_type: event.event_type.as_str(), metadata: &event.metadata, + naming_thread: event.naming_thread, observed_at, }, )?; @@ -228,9 +228,9 @@ fn upsert_device( now: i64, ) -> Result { let owner = user_id.to_string(); - let host_name = normalize_text(&event.device.host_name); - let platform = normalize_slug(&event.device.platform); - let os_version = normalized_optional(event.device.os_version.as_deref()); + let host_name = event.device.host_name.clone(); + let platform = event.device.platform.clone(); + let os_version = event.device.os_version.clone(); let existing = devices::table .filter(devices::user_id.eq(&owner)) .filter(devices::host_name.eq(&host_name)) @@ -283,7 +283,7 @@ fn upsert_session( now: i64, ) -> Result { let owner = user_id.to_string(); - let session_id = normalize_text(&event.session_id); + let session_id = event.session_id.clone(); let existing = agent_sessions::table .filter(agent_sessions::user_id.eq(&owner)) .filter(agent_sessions::agent_name.eq(agent_name)) @@ -291,10 +291,8 @@ fn upsert_session( .select(SessionRow::as_select()) .first::(connection) .optional()?; - let incoming_metadata = session_metadata_from_event(&event.metadata); if let Some(existing) = existing { let id = parse_uuid(&existing.id, "session id")?; - let metadata = merge_json(&existing.metadata, &incoming_metadata, "session metadata")?; diesel::update(agent_sessions::table.find(existing.id)) .set(( agent_sessions::device_context_id.eq(device_id.to_string()), @@ -304,7 +302,6 @@ fn upsert_session( agent_sessions::ended_at.eq(Some( existing.ended_at.unwrap_or(observed_at).max(observed_at), )), - agent_sessions::metadata.eq(metadata), agent_sessions::updated_at.eq(now), )) .execute(connection)?; @@ -322,7 +319,7 @@ fn upsert_session( session_id, started_at: observed_at, ended_at: Some(observed_at), - metadata: serialize_json(&incoming_metadata, "session metadata")?, + metadata: "{}".to_owned(), created_at: now, updated_at: now, }) @@ -330,164 +327,161 @@ fn upsert_session( Ok(id) } -fn resolve_turn_index( +/// Places an event in a turn of its session and returns the turn's id and +/// position. +/// +/// Events naming the same turn share it, and an event naming none joins the +/// turn in progress when it was observed. Turns are numbered by when they +/// started, so a turn that arrives late moves the turns after it. +fn place_in_turn( connection: &mut SqliteConnection, + user_id: Uuid, session_pk: Uuid, - event: &IngestUsageEvent, - agent_name: &str, + turn_id: Option<&str>, observed_at: i64, -) -> Result { - if let Some(attachment) = session_turn_attachment(agent_name, &event.metadata) - && let Some(turn_index) = - attached_turn_index(connection, session_pk, &attachment, observed_at)? - { - return Ok(turn_index); + now: i64, +) -> Result<(Uuid, i32), AppError> { + let session = session_pk.to_string(); + let found = match turn_id { + Some(key) => named_turn(connection, &session, key)?, + None => turn_in_progress(connection, &session, observed_at)?, + }; + let (turn_pk, turn_index, reordered) = if let Some(turn) = found { + diesel::update(agent_turns::table.find(&turn.id)) + .set(( + agent_turns::started_at.eq(turn.started_at.min(observed_at)), + agent_turns::ended_at + .eq(Some(turn.ended_at.unwrap_or(observed_at).max(observed_at))), + agent_turns::updated_at.eq(now), + )) + .execute(connection)?; + (turn.id, turn.turn_index, observed_at < turn.started_at) + } else { + let last = agent_turns::table + .filter(agent_turns::session_pk.eq(&session)) + .select(max(agent_turns::turn_index)) + .first::>(connection)?; + let id = Uuid::now_v7().to_string(); + let turn_index = last.unwrap_or(0_i32).saturating_add(1_i32); + diesel::insert_into(agent_turns::table) + .values(NewTurn { + id: id.clone(), + user_id: user_id.to_string(), + session_pk: session.clone(), + turn_index, + turn_key: turn_id.map(str::to_owned), + started_at: observed_at, + ended_at: Some(observed_at), + created_at: now, + updated_at: now, + }) + .execute(connection)?; + (id, turn_index, last.is_some()) + }; + let turn_index = if reordered && number_turns(connection, &session)? { + agent_turns::table + .find(&turn_pk) + .select(agent_turns::turn_index) + .first::(connection)? + } else { + turn_index + }; + Ok((parse_uuid(&turn_pk, "turn id")?, turn_index)) +} + +/// The turn named `key` in a session. The first named turn adopts the turn +/// that events naming none opened at the start of the session. +fn named_turn( + connection: &mut SqliteConnection, + session: &str, + key: &str, +) -> Result, AppError> { + let named = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .filter(agent_turns::turn_key.eq(key)) + .select(TurnRow::as_select()) + .first::(connection) + .optional()?; + if named.is_some() { + return Ok(named); } - // Events of one turn share its identity; an event without any identity - // is a turn of its own. - if let Some(identity) = turn_identity(event) - && let Some(existing) = existing_turn_index_for_identity(connection, session_pk, &identity)? - { - return Ok(existing); + let mut turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .select(TurnRow::as_select()) + .limit(2_i64) + .load::(connection)?; + let opening = match turns.as_slice() { + [turn] if turn.turn_key.is_none() => turns.pop(), + _ => None, + }; + if let Some(turn) = &opening { + diesel::update(agent_turns::table.find(&turn.id)) + .set(agent_turns::turn_key.eq(key)) + .execute(connection)?; } - Ok(agent_turns::table - .filter(agent_turns::session_pk.eq(session_pk.to_string())) - .select(max(agent_turns::turn_index)) - .first::>(connection)? - .unwrap_or(0_i32) - .saturating_add(1)) + Ok(opening) } -/// Finds the stored turn a Claude Code side or subagent request belongs to; -/// `None` only when the session has no turn yet. -fn attached_turn_index( +/// The turn in progress when an event naming no turn was observed: the latest +/// turn started by then, else the first turn of the session. +fn turn_in_progress( connection: &mut SqliteConnection, - session_pk: Uuid, - attachment: &SessionTurnAttachment, + session: &str, observed_at: i64, -) -> Result, AppError> { - let session = session_pk.to_string(); - if let Some(prompt_id) = &attachment.prompt_id { - // Only main-loop events anchor a prompt, and each prompt id belongs to - // one turn. Walking the session index from its latest turn usually - // stops within the current turn instead of reading every megabyte - // metadata row of the session. - let prompt_turn = sql_query( - "SELECT turn_index FROM llm_usage_events \ - WHERE session_pk = ?1 \ - AND json_extract(metadata, '$.claude_code_billing.cc_prompt_id') = ?2 \ - AND json_extract(metadata, '$.claude_turn_id') IS NOT NULL \ - ORDER BY turn_index DESC, observed_at DESC, id DESC \ - LIMIT 1", - ) - .bind::(&session) - .bind::(prompt_id) - .get_result::(connection) - .optional()?; - if let Some(row) = prompt_turn { - return Ok(Some(row.turn_index)); - } - } - // Events may arrive out of order, so the observation time picks the turn. - let preceding = agent_turns::table - .filter(agent_turns::session_pk.eq(&session)) +) -> Result, AppError> { + let started = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) .filter(agent_turns::started_at.le(observed_at)) - .order_by(( - agent_turns::started_at.desc(), - agent_turns::turn_index.desc(), - )) - .select(agent_turns::turn_index) - .first::(connection) + .order_by((agent_turns::started_at.desc(), agent_turns::id.desc())) + .select(TurnRow::as_select()) + .first::(connection) .optional()?; - if preceding.is_some() { - return Ok(preceding); + if started.is_some() { + return Ok(started); } agent_turns::table - .filter(agent_turns::session_pk.eq(&session)) - .order_by((agent_turns::started_at.asc(), agent_turns::turn_index.asc())) - .select(agent_turns::turn_index) - .first::(connection) + .filter(agent_turns::session_pk.eq(session)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select(TurnRow::as_select()) + .first::(connection) .optional() .map_err(AppError::from) } -#[derive(QueryableByName)] -struct TurnIndexRow { - #[diesel(sql_type = diesel::sql_types::Integer)] - turn_index: i32, -} - -fn existing_turn_index_for_identity( - connection: &mut SqliteConnection, - session_pk: Uuid, - identity: &TurnIdentity, -) -> Result, AppError> { - let rows = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(session_pk.to_string())) - .order_by(( - llm_usage_events::observed_at.asc(), - llm_usage_events::id.asc(), - )) - .select((llm_usage_events::turn_index, llm_usage_events::metadata)) - .load::<(i32, String)>(connection)?; - for (turn_index, metadata) in rows { - let metadata = serde_json::from_str::(&metadata).map_err(|error| { - AppError::internal(format!("stored event metadata is invalid JSON: {error}")) - })?; - if metadata - .get(identity.kind.metadata_key()) - .and_then(serde_json::Value::as_str) - == Some(identity.value.as_str()) - { - return Ok(Some(turn_index)); - } +/// Numbers the turns of a session, and their events, by when the turns +/// started. Writes nothing and returns `false` while the numbers already +/// follow that order. +fn number_turns(connection: &mut SqliteConnection, session: &str) -> Result { + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select((agent_turns::id, agent_turns::turn_index)) + .load::<(String, i32)>(connection)?; + let moved = turns + .iter() + .zip(1_i32..) + .filter(|((_, index), position)| index != position) + .collect::>(); + if moved.is_empty() { + return Ok(false); } - Ok(None) -} - -fn upsert_turn( - connection: &mut SqliteConnection, - user_id: Uuid, - session_pk: Uuid, - turn_index: i32, - observed_at: i64, - now: i64, -) -> Result { - let session = session_pk.to_string(); - let existing = agent_turns::table - .filter(agent_turns::session_pk.eq(&session)) - .filter(agent_turns::turn_index.eq(turn_index)) - .select(TurnRow::as_select()) - .first::(connection) - .optional()?; - if let Some(existing) = existing { - let id = parse_uuid(&existing.id, "turn id")?; - diesel::update(agent_turns::table.find(existing.id)) - .set(( - agent_turns::started_at.eq(existing.started_at.min(observed_at)), - agent_turns::ended_at.eq(Some( - existing.ended_at.unwrap_or(observed_at).max(observed_at), - )), - agent_turns::updated_at.eq(now), - )) + // Moved turns first step above every current number, so no single-row + // update collides with another turn's number or goes below one. + let above = turns.iter().map(|(_, index)| *index).max().unwrap_or(0_i32); + for ((id, _), position) in &moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(above.saturating_add(*position))) .execute(connection)?; - return Ok(id); } - - let id = Uuid::now_v7(); - diesel::insert_into(agent_turns::table) - .values(NewTurn { - id: id.to_string(), - user_id: user_id.to_string(), - session_pk: session, - turn_index, - started_at: observed_at, - ended_at: Some(observed_at), - created_at: now, - updated_at: now, - }) - .execute(connection)?; - Ok(id) + for ((id, _), position) in moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(position)) + .execute(connection)?; + diesel::update(llm_usage_events::table.filter(llm_usage_events::turn_pk.eq(id))) + .set(llm_usage_events::turn_index.eq(position)) + .execute(connection)?; + } + Ok(true) } /// Writes the SQLite FTS5 projection. Virtual tables are intentionally outside @@ -499,23 +493,20 @@ fn insert_search_projection( session_pk: Uuid, event: &IngestUsageEvent, ) -> Result<(), AppError> { - let projection = - SearchProjection::from_source(normalized_optional(event.text.as_deref()), &event.metadata); + let projection = SearchProjection::from_source(event.text.clone(), &event.metadata); sql_query( "INSERT INTO usage_events_fts ( event_pk, user_id, session_pk, session_id, content, tool_names, - tool_content, commands, file_paths - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + tool_content + ) VALUES (?, ?, ?, ?, ?, ?, ?)", ) .bind::(event_pk.to_string()) .bind::(user_id.to_string()) .bind::(session_pk.to_string()) - .bind::(normalize_text(&event.session_id)) + .bind::(&event.session_id) .bind::, _>(projection.content) .bind::(projection.tool_names.join("\n")) .bind::(projection.tool_content.join("\n")) - .bind::(projection.commands.join("\n")) - .bind::(projection.file_paths.join("\n")) .execute(connection)?; Ok(()) } @@ -525,14 +516,9 @@ fn ingest_one_diagnostic_capture( user_id: Uuid, capture: &IngestDiagnosticCapture, ) -> Result { - let event_ids = capture - .event_ids - .iter() - .map(|event_id| normalize_text(event_id)) - .collect::>(); let events = llm_usage_events::table .filter(llm_usage_events::user_id.eq(user_id.to_string())) - .filter(llm_usage_events::event_id.eq_any(&event_ids)) + .filter(llm_usage_events::event_id.eq_any(&capture.event_ids)) .select(( llm_usage_events::id, llm_usage_events::device_context_id, @@ -681,7 +667,7 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(timestamp_to_micros(to))); } - let agent_names = agent_names_filter(filters.agent_names); + let agent_names = exact_values_filter(filters.agent_names); if !agent_names.is_empty() { query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } @@ -695,7 +681,7 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - let providers = providers_filter(filters.llm_providers); + let providers = exact_values_filter(filters.llm_providers); if !providers.is_empty() { query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } @@ -733,8 +719,8 @@ fn load_filtered_events( llm_usage_events::llm_provider, llm_usage_events::llm_model, llm_usage_events::event_type, - sql::>("NULL"), - sql::>("NULL"), + None::.into_sql::>(), + None::>.into_sql::>(), llm_usage_events::input_tokens, llm_usage_events::output_tokens, llm_usage_events::cache_read_tokens, @@ -742,8 +728,11 @@ fn load_filtered_events( llm_usage_events::reasoning_tokens, llm_usage_events::total_tokens, llm_usage_events::observed_at, - sql::("'{}'"), + "{}".into_sql::(), llm_usage_events::created_at, + llm_usage_events::stream_id, + llm_usage_events::response_id, + llm_usage_events::previous_response_id, )) .load::(connection)? }; @@ -803,12 +792,9 @@ type OutlineRow = ( i64, i64, i64, - bool, i64, i64, Option, - Option, - Option, bool, ); @@ -836,21 +822,12 @@ pub(super) fn session_outline( llm_usage_events::cache_write_tokens, llm_usage_events::reasoning_tokens, llm_usage_events::total_tokens, - sql::("json_type(metadata, '$.token_usage_estimated') IS 'true'"), llm_usage_events::observed_at, sql::("coalesce(length(text), 0)"), sql::>(&format!( "CASE WHEN event_type = 'request' THEN substr(text, -{OUTLINE_PREVIEW_CHARS}) \ ELSE substr(text, 1, {OUTLINE_PREVIEW_CHARS}) END" )), - sql::>( - "CASE WHEN json_type(metadata, '$.response_id') = 'text' \ - THEN json_extract(metadata, '$.response_id') END", - ), - sql::>( - "CASE WHEN json_type(metadata, '$.previous_response_id') = 'text' \ - THEN json_extract(metadata, '$.previous_response_id') END", - ), // SQLite text functions stop at the first NUL character, so the // length and preview above are wrong for text containing NUL. sql::("coalesce(instr(CAST(text AS BLOB), x'00') > 0, 0)"), @@ -858,7 +835,7 @@ pub(super) fn session_outline( .load::(connection)?; let nul_text_ids = rows .iter() - .filter(|row| row.19) + .filter(|row| row.16) .map(|row| row.0.clone()) .collect::>(); let nul_texts = if nul_text_ids.is_empty() { @@ -885,7 +862,7 @@ pub(super) fn session_outline( .map(|mut row| { let attachment_count = attachment_counts.get(&row.0).copied().unwrap_or(0); if let Some(text) = nul_texts.get(&row.0) { - (row.15, row.16) = text_outline(text, &row.6)?; + (row.14, row.15) = text_outline(text, &row.6)?; } Ok(UsageEventOutline { id: parse_uuid(&row.0, "event id")?, @@ -901,12 +878,10 @@ pub(super) fn session_outline( cache_write_tokens: row.10, reasoning_tokens: row.11, total_tokens: row.12, - token_usage_estimated: row.13, - observed_at: timestamp_from_micros(row.14)?, - text_chars: row.15, - preview: row.16, + observed_at: timestamp_from_micros(row.13)?, + text_chars: row.14, + preview: row.15, attachment_count, - ordering_metadata: UsageEventOutline::ordering_metadata(row.17, row.18), }) }) .collect::, AppError>>()?; @@ -981,7 +956,7 @@ fn assemble_timeline( session_pk: session.id, user_id: session.user_id, device_context_id: session.device_context_id, - agent_name: canonical_agent_name(&session.agent_name), + agent_name: session.agent_name, agent_version: session.agent_version, session_id: session.session_id, started_at: session.started_at, @@ -1069,9 +1044,9 @@ pub(super) fn usage_summary( .map(|json| (!values.is_empty(), json)) .map_err(|error| AppError::internal(format!("serialize summary filter: {error}"))) }; - let (has_agents, agent_names) = filter_list(agent_names_filter(&query.agent_name))?; + let (has_agents, agent_names) = filter_list(exact_values_filter(&query.agent_name))?; let (has_sessions, session_ids) = filter_list(exact_values_filter(&query.session_id))?; - let (has_providers, providers) = filter_list(providers_filter(&query.llm_provider))?; + let (has_providers, providers) = filter_list(exact_values_filter(&query.llm_provider))?; let (has_models, models) = filter_list(exact_values_filter(&query.llm_model))?; let event_type = query .event_type @@ -1136,12 +1111,7 @@ const SUMMARY_AGGREGATE_SQL: &str = "\ CASE WHEN ?2 THEN d.host_name END AS host_name, \ CASE WHEN ?2 THEN d.platform END AS platform, \ CASE WHEN ?2 THEN d.os_version END AS os_version, \ - CASE \ - WHEN ?3 THEN CASE \ - WHEN e.agent_name IN ('claude', 'claude-desktop') THEN 'claude-code' \ - ELSE e.agent_name \ - END \ - END AS agent_name, \ + CASE WHEN ?3 THEN e.agent_name END AS agent_name, \ CASE WHEN ?4 THEN e.llm_provider END AS llm_provider, \ CASE WHEN ?5 THEN e.llm_model END AS llm_model, \ CASE WHEN ?6 THEN e.event_type END AS event_type, \ @@ -1404,7 +1374,7 @@ fn event_response( platform: device.map(|value| value.platform.clone()), session_pk: event.session_pk, turn_pk: event.turn_pk, - agent_name: canonical_agent_name(&event.agent_name), + agent_name: event.agent_name, agent_version: event.agent_version, session_id: event.session_id, turn_index: event.turn_index, @@ -1420,20 +1390,14 @@ fn event_response( reasoning_tokens: event.reasoning_tokens, total_tokens: event.total_tokens, observed_at: event.observed_at, + stream_id: event.stream_id, + response_id: event.response_id, + previous_response_id: event.previous_response_id, metadata: event.metadata, attachments, } } -fn merge_json(stored: &str, incoming: &serde_json::Value, field: &str) -> Result { - let mut stored = serde_json::from_str::(stored) - .map_err(|error| AppError::internal(format!("stored {field} is invalid JSON: {error}")))?; - if let (Some(stored), Some(incoming)) = (stored.as_object_mut(), incoming.as_object()) { - stored.extend(incoming.clone()); - } - serialize_json(&stored, field) -} - fn serialize_json(value: &serde_json::Value, field: &str) -> Result { serde_json::to_string(value) .map_err(|error| AppError::internal(format!("serialize {field}: {error}"))) @@ -1531,11 +1495,10 @@ mod tests { #[test] fn validation_failure_leaves_relational_and_fts_tables_empty() { let mut database = database(); - let mut request = request_with_text("invalid event"); - request.events[0].token_usage.input_tokens = Some(-1); + let request = request_with_text("invalid event"); - ingest_events(&mut database, &request, OWNER_ID, 100) - .expect_err("negative tokens should reject the batch"); + ingest_events(&mut database, &request, OWNER_ID, 0) + .expect_err("a batch over the size limit should be rejected"); let relational_count = llm_usage_events::table .select(count_star()) .first::(&mut database) @@ -1613,6 +1576,41 @@ mod tests { assert_eq!(after.turns[0].events[0].metadata, expected); } + #[test] + fn ingest_stores_reported_values_unchanged() { + let mut database = database(); + let mut request = request_with_text(" hello \n"); + let event = &mut request.events[0]; + event.session_id = " Session ".to_owned(); + event.llm.provider = "Gateway_Host".to_owned(); + event.llm.model = " Model ".to_owned(); + event.device.platform = "macOS".to_owned(); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + let mut empty = request_with_text(""); + empty.events[0].event_id = "sqlite-empty-text".to_owned(); + ingest_events(&mut database, &empty, OWNER_ID, 100).expect("ingest should succeed"); + + let query: RawEventsQuery = + serde_json::from_value(json!({})).expect("query should deserialize"); + let events = raw_events(&mut database, &query, OWNER_ID, 50) + .expect("events should load") + .events; + let stored = events + .iter() + .find(|event| event.event_id == "sqlite-event") + .expect("the first event should be stored"); + assert_eq!(stored.session_id, " Session "); + assert_eq!(stored.llm_provider, "Gateway_Host"); + assert_eq!(stored.llm_model, " Model "); + assert_eq!(stored.platform.as_deref(), Some("macOS")); + assert_eq!(stored.text.as_deref(), Some(" hello \n")); + let empty = events + .iter() + .find(|event| event.event_id == "sqlite-empty-text") + .expect("the empty-text event should be stored"); + assert_eq!(empty.text.as_deref(), Some("")); + } + #[test] fn event_list_leaves_out_content_but_keeps_what_a_list_shows() { let mut database = summary_seeded(); @@ -1642,10 +1640,10 @@ mod tests { fn summary_seeded() -> SqliteConnection { let mut database = database(); let event = |id: &str, agent: &str, session: &str, turn: i32, observed_at: &str| { - let (provider, model) = if agent == "Codex" { - ("OpenAI", "gpt-5") + let (provider, model) = if agent == "codex" { + ("openai", "gpt-5") } else { - ("Anthropic", "claude-x") + ("anthropic", "claude-x") }; json!({ "event_id": id, @@ -1653,24 +1651,25 @@ mod tests { "device": {"host_name": "alice-mbp", "platform": "macos", "os_version": "15.1"}, "agent": {"name": agent}, "session_id": session, + "turn_id": format!("{session}-{turn}"), "llm": {"provider": provider, "model": model}, "event_type": if id.ends_with("req") { "request" } else { "response" }, "text": "body", "token_usage": if id.ends_with("req") { - json!({"input_tokens": 10_i64, "cache_read_tokens": 4_i64}) + json!({"input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 4_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 14_i64}) } else { - json!({"output_tokens": 5_i64}) + json!({"input_tokens": 0_i64, "output_tokens": 5_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 5_i64}) }, - "metadata": {"raw_body": "large", "turn_id": format!("{session}-{turn}")} + "metadata": {"raw_body": "large"} }) }; let request: IngestEventsRequest = serde_json::from_value(json!({ "events": [ - event("a-req", "Codex", "alpha", 1_i32, "2026-09-01T23:00:00Z"), - event("a-res", "Codex", "alpha", 1_i32, "2026-09-01T23:30:00Z"), + event("a-req", "codex", "alpha", 1_i32, "2026-09-01T23:00:00Z"), + event("a-res", "codex", "alpha", 1_i32, "2026-09-01T23:30:00Z"), event("b-req", "claude-desktop", "beta", 1_i32, "2026-09-02T01:00:00Z"), event("c-res", "claude-code", "gamma", 1_i32, "2026-09-02T02:00:00Z"), - event("a2-req", "Codex", "alpha", 2_i32, "2026-09-03T00:00:00Z"), + event("a2-req", "codex", "alpha", 2_i32, "2026-09-03T00:00:00Z"), ] })) .expect("request should deserialize"); @@ -1686,7 +1685,7 @@ mod tests { } #[test] - fn summary_groups_in_sql_with_canonical_agents_and_distinct_counts() { + fn summary_groups_in_sql_by_reported_agent_and_distinct_counts() { let mut database = summary_seeded(); let rows = summary( @@ -1694,22 +1693,29 @@ mod tests { json!({"group_by": "agent,provider,model", "to": "2026-09-03T00:00:00Z"}), ); + // Claude Desktop is its own Agent, not an alias of Claude Code. assert_eq!( rows, json!([ - // Equal totals fall back to the Agent name. - { - "agent_name": "claude-code", "llm_provider": "anthropic", - "llm_model": "claude-x", - "sessions": 2_i64, "turns": 2_i64, "requests": 1_i64, "responses": 1_i64, - "input_tokens": 10_i64, "output_tokens": 5_i64, "cache_read_tokens": 4_i64, - "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 19_i64 - }, { "agent_name": "codex", "llm_provider": "openai", "llm_model": "gpt-5", "sessions": 1_i64, "turns": 1_i64, "requests": 1_i64, "responses": 1_i64, "input_tokens": 10_i64, "output_tokens": 5_i64, "cache_read_tokens": 4_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 19_i64 + }, + { + "agent_name": "claude-desktop", "llm_provider": "anthropic", + "llm_model": "claude-x", + "sessions": 1_i64, "turns": 1_i64, "requests": 1_i64, "responses": 0_i64, + "input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 4_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 14_i64 + }, + { + "agent_name": "claude-code", "llm_provider": "anthropic", + "llm_model": "claude-x", + "sessions": 1_i64, "turns": 1_i64, "requests": 0_i64, "responses": 1_i64, + "input_tokens": 0_i64, "output_tokens": 5_i64, "cache_read_tokens": 0_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 5_i64 } ]) ); @@ -1721,7 +1727,11 @@ mod tests { let rows = summary( &mut database, - json!({"group_by": "day,event_type", "agent_name": ["claude"], "limit": 10_i64}), + json!({ + "group_by": "day,event_type", + "agent_name": ["claude-code", "claude-desktop"], + "limit": 10_i64, + }), ); let mut buckets = rows @@ -1751,12 +1761,30 @@ mod tests { assert_eq!(codex_requests[0]["requests"], json!(2_i64)); assert_eq!(codex_requests[0]["turns"], json!(2_i64)); assert_eq!(codex_requests.as_array().map(Vec::len), Some(1)); + // Agent names match exactly: no alias expansion and no case folding. + for agent in ["claude", "Codex"] { + let rows = summary( + &mut database, + json!({"group_by": "agent", "agent_name": [agent]}), + ); + assert_eq!(rows, json!([]), "{agent} must not match another Agent"); + } + // Providers match exactly as reported, without case folding. + let folded = summary( + &mut database, + json!({"group_by": "agent", "llm_provider": ["OpenAI"]}), + ); + assert_eq!( + folded, + json!([]), + "OpenAI must not match the reported openai" + ); // A list matches any of its values. let either = summary( &mut database, json!({ "group_by": "agent", - "llm_provider": ["OpenAI", "Anthropic"], + "llm_provider": ["openai", "anthropic"], "session_id": ["alpha", "beta"], }), ); @@ -1770,7 +1798,7 @@ mod tests { assert_eq!( agents, vec![ - (json!("claude-code"), json!(1_i64)), + (json!("claude-desktop"), json!(1_i64)), (json!("codex"), json!(1_i64)), ] ); @@ -1792,27 +1820,21 @@ mod tests { assert!(rows[0].get("agent_name").is_none()); } - /// One request of `agent` observed `second` seconds into a fixed minute. - fn turn_event( - id: &str, - agent: &str, - second: u32, - metadata: serde_json::Value, - ) -> serde_json::Value { - let mut event = json!({ + /// One request observed `second` seconds into a fixed minute, working for + /// `turn` when it names one. + fn turn_event(id: &str, agent: &str, second: u32, turn: Option<&str>) -> serde_json::Value { + json!({ "event_id": id, "observed_at": format!("2026-10-08T00:00:{second:02}Z"), "device": {"host_name": "local", "platform": "macos"}, "agent": {"name": agent}, "session_id": "thread-session", - // Distinct collector indexes expose any event left on its own turn. - "turn_index": second, + "turn_id": turn, "llm": {"provider": "Anthropic", "model": "claude-x"}, "event_type": "request", - "token_usage": {"input_tokens": 1_i64} - }); - event["metadata"] = metadata; - event + "token_usage": {"input_tokens": 1_i64, "output_tokens": 0_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64}, + "metadata": {} + }) } fn ingest_turn_events(database: &mut SqliteConnection, events: Vec) { @@ -1830,140 +1852,144 @@ mod tests { .expect("event should be stored") } - fn main_loop(turn: u32, prompt: &str) -> serde_json::Value { - json!({ - "claude_turn_id": format!("cc_turn:{turn}"), - "claude_code_billing": {"cc_prompt_id": prompt, "cc_turn_index": turn.to_string()} - }) + /// Stored turns in number order: key, number, start and end second. + fn turns(database: &mut SqliteConnection) -> Vec<(Option, i32, i64, Option)> { + agent_turns::table + .order_by(agent_turns::turn_index.asc()) + .select(( + agent_turns::turn_key, + agent_turns::turn_index, + agent_turns::started_at, + agent_turns::ended_at, + )) + .load::<(Option, i32, i64, Option)>(database) + .expect("turns should load") + } + + fn second(value: i64) -> i64 { + super::timestamp_to_micros( + chrono::DateTime::parse_from_rfc3339(&format!("2026-10-08T00:00:{value:02}Z")) + .expect("timestamp should parse") + .into(), + ) } #[test] - fn claude_code_side_and_subagent_requests_join_the_turn_they_occur_in() { + fn events_naming_a_turn_share_it_and_others_join_the_turn_in_progress() { let mut database = database(); ingest_turn_events( &mut database, vec![ - turn_event("main-1", "claude-code", 10, main_loop(1, "prompt-1")), - turn_event( - "subagent-1", - "claude-code", - 12, - json!({"claude_code_billing": {"cc_prompt_id": "prompt-1", "cc_is_subagent": "true"}}), - ), - turn_event("main-2", "claude-code", 30, main_loop(2, "prompt-2")), - // A failed `continue` replays with a new prompt id, same turn. - turn_event( - "main-2-replay", - "claude-code", - 35, - main_loop(2, "prompt-2b"), - ), - turn_event( - "subagent-2", - "claude-code", - 40, - json!({"claude_code_billing": {"cc_prompt_id": "prompt-2b", "cc_is_subagent": "true"}}), - ), - // A recap between the turns carries no billing header. - turn_event( - "recap", - "claude-code", - 20, - json!({"message_id": "msg-recap"}), - ), + turn_event("main-1", "codex", 10, Some("p1")), + turn_event("subagent-1", "codex", 12, Some("p1")), + turn_event("recap", "codex", 20, None), + turn_event("main-2", "codex", 30, Some("p2")), + turn_event("subagent-2", "codex", 40, Some("p2")), ], ); - // Out of order: observed during turn 2, delivered after everything. + // Delivered late: a request naming no turn observed during turn 2, and + // a background agent of turn 1 still working during turn 2. ingest_turn_events( &mut database, vec![ - turn_event( - "late-title", - "claude-code", - 32, - json!({"request_hash": "title"}), - ), - turn_event( - "late-subagent", - "claude-code", - 45, - json!({"claude_code_billing": {"cc_prompt_id": "prompt-1"}}), - ), + turn_event("late-title", "codex", 32, None), + turn_event("late-subagent", "codex", 45, Some("p1")), ], ); - let main_1 = turn_of(&mut database, "main-1"); - let main_2 = turn_of(&mut database, "main-2"); - assert_ne!(main_1, main_2); - assert_eq!(turn_of(&mut database, "main-2-replay"), main_2); - assert_eq!(turn_of(&mut database, "subagent-1"), main_1); - assert_eq!(turn_of(&mut database, "subagent-2"), main_2); - assert_eq!(turn_of(&mut database, "recap"), main_1); - assert_eq!(turn_of(&mut database, "late-title"), main_2); - assert_eq!(turn_of(&mut database, "late-subagent"), main_1); - let turns = agent_turns::table - .order_by(agent_turns::turn_index.asc()) - .select(( - agent_turns::turn_index, - agent_turns::started_at, - agent_turns::ended_at, - )) - .load::<(i32, i64, Option)>(&mut database) - .expect("turns should load"); - let second = |value: i64| { - super::timestamp_to_micros( - chrono::DateTime::parse_from_rfc3339(&format!("2026-10-08T00:00:{value:02}Z")) - .expect("timestamp should parse") - .into(), - ) - }; + for (event, turn) in [ + ("main-1", 1_i32), + ("subagent-1", 1_i32), + ("recap", 1_i32), + ("late-subagent", 1_i32), + ("main-2", 2_i32), + ("subagent-2", 2_i32), + ("late-title", 2_i32), + ] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } assert_eq!( - turns, + turns(&mut database), vec![ - (main_1, second(10), Some(second(45))), - (main_2, second(30), Some(second(40))), + (Some("p1".to_owned()), 1_i32, second(10), Some(second(45))), + (Some("p2".to_owned()), 2_i32, second(30), Some(second(40))), ] ); } #[test] - fn claude_code_side_request_before_any_turn_keeps_todays_turn() { + fn the_first_named_turn_adopts_the_turn_opened_without_one() { let mut database = database(); ingest_turn_events( &mut database, - vec![turn_event( - "title", - "claude-code", - 5, - json!({"request_hash": "title"}), - )], + vec![turn_event("probe", "claude-code", 5, None)], ); - let first = turn_of(&mut database, "title"); ingest_turn_events( &mut database, vec![ - turn_event("main-1", "claude-code", 10, main_loop(1, "prompt-1")), - // Earlier than every turn: joins the session's earliest turn. - turn_event("probe", "claude-code", 1, json!({})), + turn_event("main-1", "claude-code", 10, Some("p1")), + turn_event("main-2", "claude-code", 30, Some("p2")), + // Earlier than every turn: joins the first one. + turn_event("early", "claude-code", 1, None), ], ); - assert_ne!(turn_of(&mut database, "main-1"), first); - assert_eq!(turn_of(&mut database, "probe"), first); + + assert_eq!(turn_of(&mut database, "probe"), 1_i32); + assert_eq!(turn_of(&mut database, "main-1"), 1_i32); + assert_eq!(turn_of(&mut database, "early"), 1_i32); + assert_eq!(turn_of(&mut database, "main-2"), 2_i32); + assert_eq!( + turns(&mut database), + vec![ + (Some("p1".to_owned()), 1_i32, second(1), Some(second(10))), + (Some("p2".to_owned()), 2_i32, second(30), Some(second(30))), + ] + ); } #[test] - fn other_agents_keep_provider_turn_identities() { + fn turns_are_numbered_by_start_time_whatever_order_they_arrive_in() { let mut database = database(); ingest_turn_events( &mut database, vec![ - turn_event("codex-1", "codex", 10, json!({"codex_turn_id": "turn-a"})), - turn_event("codex-2", "codex", 20, json!({"response_id": "resp-b"})), + turn_event("b", "codex", 30, Some("b")), + turn_event("b-next", "codex", 31, Some("b")), + turn_event("c", "codex", 50, Some("c")), ], ); - assert_ne!( - turn_of(&mut database, "codex-1"), - turn_of(&mut database, "codex-2") + assert_eq!(turn_of(&mut database, "c"), 2_i32); + + // A turn delivered late started first. + ingest_turn_events(&mut database, vec![turn_event("a", "codex", 10, Some("a"))]); + for (event, turn) in [("a", 1_i32), ("b", 2_i32), ("b-next", 2_i32), ("c", 3_i32)] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } + + // A late event of a known turn moves its start before every other turn. + ingest_turn_events( + &mut database, + vec![turn_event("c-early", "codex", 5, Some("c"))], + ); + for (event, turn) in [ + ("c-early", 1_i32), + ("c", 1_i32), + ("a", 2_i32), + ("b", 3_i32), + ("b-next", 3_i32), + ] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } + assert_eq!( + turns(&mut database) + .into_iter() + .map(|(key, index, ..)| (key, index)) + .collect::>(), + vec![ + (Some("c".to_owned()), 1_i32), + (Some("a".to_owned()), 2_i32), + (Some("b".to_owned()), 3_i32), + ] ); } @@ -1973,14 +1999,13 @@ mod tests { "event_id": "sqlite-event", "observed_at": "2026-08-27T00:00:00Z", "device": {"host_name": "local", "platform": "macos"}, - "agent": {"name": "Codex"}, + "agent": {"name": "codex"}, "session_id": "sqlite-session", - "turn_index": 1, - "llm": {"provider": "OpenAI", "model": "gpt-5"}, + "llm": {"provider": "openai", "model": "gpt-5"}, "event_type": "response", "text": text, - "token_usage": {"output_tokens": 3}, - "metadata": {"response_id": "sqlite-response"} + "token_usage": {"input_tokens": 0_i64, "output_tokens": 3_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 3_i64}, + "metadata": {} }] })) .expect("test ingest request should deserialize") diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs index cc9f761..74390a9 100644 --- a/crates/abyss-backend/src/storage/sqlite/session_list.rs +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -213,12 +213,12 @@ mod tests { "device": {"host_name": "alice-mbp", "platform": "macos"}, "agent": {"name": agent}, "session_id": session, - "turn_index": turn, + "turn_id": format!("{session}-{turn}"), "llm": {"provider": "OpenAI", "model": model}, "event_type": "response", "text": "reply", - "token_usage": {"output_tokens": 10}, - "metadata": {"response_id": id} + "token_usage": {"input_tokens": 0_i64, "output_tokens": 10_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 10_i64}, + "metadata": {} }) } @@ -226,10 +226,10 @@ mod tests { let mut database = database(); let request: IngestEventsRequest = serde_json::from_value(json!({ "events": [ - event("a1", "alpha", "Codex", "2026-09-01T00:00:00Z", 1), - event_with_model("a2", "alpha", "Codex", "2026-09-03T00:00:00Z", 2, "gpt-5-mini"), + event("a1", "alpha", "codex", "2026-09-01T00:00:00Z", 1), + event_with_model("a2", "alpha", "codex", "2026-09-03T00:00:00Z", 2, "gpt-5-mini"), event("b1", "beta", "claude-desktop", "2026-09-02T00:00:00Z", 1), - event("c1", "gamma", "Codex", "2026-08-01T00:00:00Z", 1), + event("c1", "gamma", "codex", "2026-08-01T00:00:00Z", 1), ] })) .expect("request should deserialize"); @@ -272,15 +272,23 @@ mod tests { } #[test] - fn filters_by_agent_alias_and_activity_window() { + fn filters_by_exact_agent_and_activity_window() { let mut database = seeded(); - let claude = SessionListQuery { - agent_name: vec!["Claude Code".to_owned()], + let desktop = SessionListQuery { + agent_name: vec!["claude-desktop".to_owned()], ..SessionListQuery::default() }; - assert_eq!(list(&mut database, claude), [("beta".to_owned(), 1, 1)]); + assert_eq!(list(&mut database, desktop), [("beta".to_owned(), 1, 1)]); + // Claude Desktop is not an alias of Claude Code, and case is not folded. + for agent in ["claude-code", "Codex"] { + let other = SessionListQuery { + agent_name: vec![agent.to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, other), Vec::new(), "{agent}"); + } let either = SessionListQuery { - agent_name: vec!["Claude Code".to_owned(), "codex".to_owned()], + agent_name: vec!["claude-desktop".to_owned(), "codex".to_owned()], ..SessionListQuery::default() }; assert_eq!(list(&mut database, either).len(), 3); diff --git a/crates/abyss-backend/src/usage/event_order.rs b/crates/abyss-backend/src/usage/event_order.rs index 64e749c..4d4421e 100644 --- a/crates/abyss-backend/src/usage/event_order.rs +++ b/crates/abyss-backend/src/usage/event_order.rs @@ -1,21 +1,13 @@ //! Deterministic ordering for normalized events inside Agent timeline turns. //! -//! Codex emits a provider response chain through `response_id` and -//! `previous_response_id`. Timeline ordering prefers that native relationship, -//! then falls back to high-precision observation time, event side, and stable -//! event identity. - -use std::{ - cmp::Ordering, - collections::{BTreeMap, HashMap, HashSet}, -}; +//! Events are ordered by turn, high-precision observation time, event side, +//! and stable event identity. use chrono::{DateTime, Utc}; /// Event fields required by deterministic timeline ordering. pub trait TimelineEvent { fn turn_index(&self) -> i32; - fn metadata(&self) -> &serde_json::Value; fn observed_at(&self) -> DateTime; fn event_type(&self) -> &str; fn event_id(&self) -> &str; @@ -27,10 +19,6 @@ impl TimelineEvent for crate::db::models::UsageEvent { self.turn_index } - fn metadata(&self) -> &serde_json::Value { - &self.metadata - } - fn observed_at(&self) -> DateTime { self.observed_at } @@ -49,10 +37,6 @@ impl TimelineEvent for super::UsageEventOutline { self.turn_index } - fn metadata(&self) -> &serde_json::Value { - &self.ordering_metadata - } - fn observed_at(&self) -> DateTime { self.observed_at } @@ -70,426 +54,136 @@ impl TimelineEvent for super::UsageEventOutline { pub struct UsageEventTimelineOrder; impl UsageEventTimelineOrder { - /// Sorts events by turn, provider chain, observation time, side, and ID. + /// Sorts events by turn, observation time, side, and ID. /// /// The final event identifier tie-breaker makes output deterministic even - /// for corrupt cycles or collectors with identical timestamps. + /// for collectors with identical timestamps. pub fn sort(events: &mut [T]) { - let provider_ranks = ProviderResponseRanks::from_events(events); events.sort_by(|left, right| { - left.turn_index().cmp(&right.turn_index()).then_with(|| { - provider_ranks.compare_events(left, right).then_with(|| { - left.observed_at() - .cmp(&right.observed_at()) - .then_with(|| { - event_side_rank(left.event_type()) - .cmp(&event_side_rank(right.event_type())) - }) - .then_with(|| left.event_id().cmp(right.event_id())) + left.turn_index() + .cmp(&right.turn_index()) + .then_with(|| left.observed_at().cmp(&right.observed_at())) + .then_with(|| { + event_side_rank(left.event_type()).cmp(&event_side_rank(right.event_type())) }) - }) + .then_with(|| left.event_id().cmp(right.event_id())) }); } } -struct ProviderResponseRanks { - ranks: HashMap<(i32, String), usize>, - native_turns: HashSet, -} - -impl ProviderResponseRanks { - fn from_events(events: &[T]) -> Self { - let mut nodes_by_turn = BTreeMap::>::new(); - // Native provider ordering is safe only when every event in a turn has - // a response_id. A partially instrumented turn falls back as a whole so - // ranked and unranked events cannot interleave unpredictably. - let mut native_turns = events - .iter() - .map(TimelineEvent::turn_index) - .collect::>(); - for event in events { - let Some(response_id) = metadata_string(event.metadata(), "response_id") else { - native_turns.remove(&event.turn_index()); - continue; - }; - let previous_response_id = - metadata_string(event.metadata(), "previous_response_id").map(str::to_owned); - let nodes = nodes_by_turn.entry(event.turn_index()).or_default(); - let node = - nodes - .entry(response_id.to_owned()) - .or_insert_with(|| ProviderResponseNode { - response_id: response_id.to_owned(), - previous_response_id: previous_response_id.clone(), - first_observed_at: event.observed_at(), - first_event_id: event.event_id().to_owned(), - }); - node.merge_evidence(event, previous_response_id); - } - - let mut ranks = HashMap::new(); - for (turn_index, nodes) in nodes_by_turn { - Self::rank_turn(turn_index, &nodes, &mut ranks); - } - Self { - ranks, - native_turns, - } +const fn event_side_rank(event_type: &str) -> u8 { + match event_type.as_bytes() { + b"request" => 0, + b"response" => 1, + _ => 2, } +} - fn rank_turn( - turn_index: i32, - nodes: &HashMap, - ranks: &mut HashMap<(i32, String), usize>, - ) { - let mut roots = Vec::new(); - let mut children = HashMap::>::new(); - for node in nodes.values() { - match node - .previous_response_id - .as_ref() - .filter(|previous| previous.as_str() != node.response_id) - .filter(|previous| nodes.contains_key(previous.as_str())) - { - Some(previous) => children - .entry(previous.clone()) - .or_default() - .push(node.response_id.clone()), - None => roots.push(node.response_id.clone()), - } - } - - // Branches can occur after retries or malformed evidence. Sort roots and - // siblings by stable observed/event fallback data before traversal. - let compare_ids = |left: &String, right: &String| { - nodes - .get(left) - .expect("provider response id should resolve") - .fallback_cmp( - nodes - .get(right) - .expect("provider response id should resolve"), - ) - }; - roots.sort_by(compare_ids); - for child_ids in children.values_mut() { - child_ids.sort_by(compare_ids); - } - - let mut visited = HashSet::new(); - let mut next_rank = 0_usize; - for root in roots { - Self::rank_chain( - turn_index, - root, - &children, - ranks, - &mut visited, - &mut next_rank, - ); - } +#[cfg(test)] +mod tests { + use chrono::{DateTime, TimeZone as _, Timelike as _, Utc}; - // Cycles have no root. Traversing remaining nodes after rooted chains - // guarantees every response still receives a deterministic rank. - let mut remaining = nodes - .keys() - .filter(|response_id| !visited.contains(response_id.as_str())) - .cloned() - .collect::>(); - remaining.sort_by(compare_ids); - for response_id in remaining { - Self::rank_chain( - turn_index, - response_id, - &children, - ranks, - &mut visited, - &mut next_rank, - ); - } - } + use super::{TimelineEvent, UsageEventTimelineOrder}; - fn rank_chain( + struct Sample { turn_index: i32, - root: String, - children: &HashMap>, - ranks: &mut HashMap<(i32, String), usize>, - visited: &mut HashSet, - next_rank: &mut usize, - ) { - let mut pending = vec![root]; - while let Some(response_id) = pending.pop() { - if !visited.insert(response_id.clone()) { - continue; - } - ranks.insert((turn_index, response_id.clone()), *next_rank); - *next_rank = next_rank.saturating_add(1); - if let Some(child_ids) = children.get(&response_id) { - pending.extend(child_ids.iter().rev().cloned()); - } - } + observed_at: DateTime, + event_type: &'static str, + event_id: &'static str, } - fn compare_events(&self, left: &T, right: &T) -> Ordering { - if left.turn_index() != right.turn_index() - || !self.native_turns.contains(&left.turn_index()) - { - return Ordering::Equal; + impl TimelineEvent for Sample { + fn turn_index(&self) -> i32 { + self.turn_index } - let left_response_id = metadata_string(left.metadata(), "response_id"); - let right_response_id = metadata_string(right.metadata(), "response_id"); - - // Request and response observations that describe the same provider - // call remain adjacent and request-first regardless of timestamp ties. - if left_response_id.is_some() && left_response_id == right_response_id { - return event_side_rank(left.event_type()).cmp(&event_side_rank(right.event_type())); + fn observed_at(&self) -> DateTime { + self.observed_at } - let native_order = left_response_id - .and_then(|response_id| self.ranks.get(&(left.turn_index(), response_id.to_owned()))) - .zip(right_response_id.and_then(|response_id| { - self.ranks - .get(&(right.turn_index(), response_id.to_owned())) - })) - .map(|(left_rank, right_rank)| left_rank.cmp(right_rank)); - if let Some(ordering) = native_order.filter(|ordering| !ordering.is_eq()) { - return ordering; + fn event_type(&self) -> &str { + self.event_type } - Ordering::Equal - } -} - -struct ProviderResponseNode { - response_id: String, - previous_response_id: Option, - first_observed_at: DateTime, - first_event_id: String, -} - -impl ProviderResponseNode { - fn merge_evidence( - &mut self, - event: &T, - previous_response_id: Option, - ) { - if self.previous_response_id.is_none() { - self.previous_response_id = previous_response_id; - } - if (event.observed_at(), event.event_id()) - < (self.first_observed_at, self.first_event_id.as_str()) - { - self.first_observed_at = event.observed_at(); - event.event_id().clone_into(&mut self.first_event_id); + fn event_id(&self) -> &str { + self.event_id } } - fn fallback_cmp(&self, other: &Self) -> Ordering { - self.first_observed_at - .cmp(&other.first_observed_at) - .then_with(|| self.first_event_id.cmp(&other.first_event_id)) - .then_with(|| self.response_id.cmp(&other.response_id)) + fn timestamp() -> DateTime { + Utc.with_ymd_and_hms(2026, 8, 4, 2, 12, 24) + .single() + .expect("timestamp should be valid") } -} - -fn metadata_string<'a>(metadata: &'a serde_json::Value, key: &str) -> Option<&'a str> { - metadata - .get(key) - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) -} -const fn event_side_rank(event_type: &str) -> u8 { - match event_type.as_bytes() { - b"request" => 0, - b"response" => 1, - _ => 2, + fn event( + turn_index: i32, + observed_at: DateTime, + event_type: &'static str, + event_id: &'static str, + ) -> Sample { + Sample { + turn_index, + observed_at, + event_type, + event_id, + } } -} - -#[cfg(all(test, feature = "postgres-es"))] -mod tests { - use chrono::{TimeZone as _, Timelike as _, Utc}; - use serde_json::json; - use uuid::Uuid; - - use super::UsageEventTimelineOrder; - use crate::db::models::UsageEvent; - - #[test] - fn native_response_chain_precedes_timestamps_and_collector_ordinals() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event( - "b-response", - "response", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 1_i64}), - ), - usage_event( - "a-response", - "response", - timestamp, - json!({"response_id": "resp_a", "provider_call_index": 2_i64}), - ), - usage_event( - "b-request", - "request", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 1_i64}), - ), - usage_event( - "a-request", - "request", - timestamp, - json!({"response_id": "resp_a", "provider_call_index": 2_i64}), - ), - ]; + fn sorted_ids(mut events: Vec) -> Vec<&'static str> { UsageEventTimelineOrder::sort(&mut events); - - assert_eq!( - event_ids(&events), - ["a-request", "a-response", "b-request", "b-response"] - ); + events.iter().map(|event| event.event_id).collect() } #[test] - fn timestamp_precedes_collector_ordinal_when_native_chain_is_incomplete() { - let first_timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let second_timestamp = first_timestamp + chrono::TimeDelta::microseconds(1); - let mut events = vec![ - usage_event( - "later-despite-smaller-ordinal", - "response", - second_timestamp, - json!({"provider_call_index": 1_i64, "response_id": "resp_later"}), - ), - usage_event( - "earlier-despite-larger-ordinal", - "response", - first_timestamp, - json!({"provider_call_index": 2_i64}), - ), - ]; - - UsageEventTimelineOrder::sort(&mut events); - + fn turn_precedes_observation_time() { + let first = timestamp(); + let second = first + chrono::TimeDelta::microseconds(1); assert_eq!( - event_ids(&events), - [ - "earlier-despite-larger-ordinal", - "later-despite-smaller-ordinal" - ] + sorted_ids(vec![ + event(2, first, "request", "turn-2"), + event(1, second, "request", "turn-1"), + ]), + ["turn-1", "turn-2"] ); } #[test] - fn high_precision_timestamp_orders_legacy_events_without_provider_evidence() { - let first = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid") + fn high_precision_timestamp_orders_events_within_a_turn() { + let first = timestamp() .with_nanosecond(1_000) .expect("microsecond should be valid"); let second = first .with_nanosecond(2_000) .expect("microsecond should be valid"); - let mut events = vec![ - usage_event("second", "request", second, json!({})), - usage_event("first", "response", first, json!({})), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["first", "second"]); + assert_eq!( + sorted_ids(vec![ + event(1, second, "request", "second"), + event(1, first, "response", "first"), + ]), + ["first", "second"] + ); } #[test] - fn request_precedes_response_for_legacy_timestamp_ties() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event("response", "response", timestamp, json!({})), - usage_event("request", "request", timestamp, json!({})), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["request", "response"]); + fn request_precedes_response_for_timestamp_ties() { + assert_eq!( + sorted_ids(vec![ + event(1, timestamp(), "response", "response"), + event(1, timestamp(), "request", "request"), + ]), + ["request", "response"] + ); } #[test] - fn broken_cycles_fall_back_to_deterministic_event_identity() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event( - "second", - "response", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 2_i64}), - ), - usage_event( - "first", - "response", - timestamp, - json!({"response_id": "resp_a", "previous_response_id": "resp_b", "provider_call_index": 1_i64}), - ), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["first", "second"]); - } - - fn usage_event( - event_id: &str, - event_type: &str, - observed_at: chrono::DateTime, - metadata: serde_json::Value, - ) -> UsageEvent { - UsageEvent { - id: Uuid::nil(), - user_id: Uuid::nil(), - device_context_id: Uuid::nil(), - session_pk: Uuid::nil(), - turn_pk: Uuid::nil(), - event_id: event_id.to_owned(), - agent_name: "codex".to_owned(), - agent_version: None, - session_id: "session".to_owned(), - turn_index: 1, - llm_provider: "openai".to_owned(), - llm_model: "gpt-test".to_owned(), - event_type: event_type.to_owned(), - text: None, - text_sha256: None, - input_tokens: 0, - output_tokens: 0, - cache_read_tokens: 0, - cache_write_tokens: 0, - reasoning_tokens: 0, - total_tokens: 0, - observed_at, - metadata, - created_at: observed_at, - } - } - - fn event_ids(events: &[UsageEvent]) -> Vec<&str> { - events.iter().map(|event| event.event_id.as_str()).collect() + fn event_identity_breaks_remaining_ties() { + assert_eq!( + sorted_ids(vec![ + event(1, timestamp(), "response", "b"), + event(1, timestamp(), "response", "a"), + ]), + ["a", "b"] + ); } } diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 035f9b8..a71b0c2 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -22,8 +22,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use uuid::Uuid; -const WORKING_DIRECTORY_METADATA_KEY: &str = "working_directory"; - #[derive(Deserialize)] /// Atomic batch accepted by the event-ingestion endpoint. pub struct IngestEventsRequest { @@ -47,13 +45,29 @@ pub struct IngestUsageEvent { pub agent: AgentPayload, /// Agent-native session identifier. pub session_id: String, + /// The user turn this request works for, unique within its session; + /// absent when the request works for no particular turn. + pub turn_id: Option, + /// Whether the request comes from a thread the Agent opened only to name + /// another session. + #[serde(default)] + pub naming_thread: bool, + /// The thread that sent this exchange's request, as the Agent declares + /// it; requests with the same value continue one conversation. + pub stream_id: Option, + /// Provider identity of the response this exchange produced; both events + /// of the exchange carry it. + pub response_id: Option, + /// The response this exchange's request continues after; the request then + /// carries only the messages added since. + pub previous_response_id: Option, /// LLM provider and model labels. pub llm: LlmPayload, /// Whether the event represents the request or response side of a call. pub event_type: UsageEventType, /// Optional prompt or response text. pub text: Option, - /// Provider-reported token counters; missing counters default to zero. + /// Token counters of the `AgentEvent`; absent counters are stored as zero. #[serde(default)] pub token_usage: TokenUsagePayload, /// Collector metadata retained with the raw event. @@ -67,10 +81,9 @@ pub struct IngestUsageEvent { #[derive(Debug, Deserialize)] /// Collector-supplied host identity used to group events by device context. pub struct DevicePayload { - /// User-visible host name; `hostname` remains an accepted JSON alias. - #[serde(alias = "hostname")] + /// User-visible host name, stored exactly as received. pub host_name: String, - /// Normalized operating-system or runtime platform label. + /// Operating-system or runtime platform label, stored exactly as received. pub platform: String, /// Optional operating-system version observed by the collector. pub os_version: Option, @@ -79,7 +92,7 @@ pub struct DevicePayload { #[derive(Debug, Deserialize)] /// Collector-supplied Agent identity. pub struct AgentPayload { - /// Agent name, normalized to a canonical slug during ingest. + /// Agent name identified by the collector, stored exactly as received. pub name: String, /// Optional Agent build or release version. pub version: Option, @@ -88,7 +101,7 @@ pub struct AgentPayload { #[derive(Debug, Deserialize)] /// Collector-supplied LLM target identity. pub struct LlmPayload { - /// Provider name, normalized to a lowercase slug during ingest. + /// Provider name, stored exactly as received. pub provider: String, /// Provider-specific model identifier. pub model: String, @@ -119,7 +132,7 @@ impl UsageEventType { clippy::struct_field_names, reason = "Public JSON token usage fields intentionally include the token suffix." )] -/// Token counters reported by the event producer. +/// Token counters of the `AgentEvent`, stored as sent. /// /// Input, cache read, cache write, and output counters are disjoint; /// reasoning is a breakdown of output. @@ -134,7 +147,7 @@ pub struct TokenUsagePayload { pub cache_write_tokens: Option, /// Part of output tokens spent on reasoning. pub reasoning_tokens: Option, - /// Sum of the four disjoint counters, derived when omitted. + /// Sum of the four disjoint counters. pub total_tokens: Option, } @@ -235,7 +248,7 @@ pub struct SummaryRow { /// Device OS version when grouped by device. #[serde(skip_serializing_if = "Option::is_none")] pub os_version: Option, - /// Canonical Agent name when grouped by Agent. + /// Agent name as reported by the collector, when grouped by Agent. #[serde(skip_serializing_if = "Option::is_none")] pub agent_name: Option, /// LLM provider when grouped by provider. @@ -438,7 +451,7 @@ pub struct UsageEventListItem { pub session_pk: Uuid, /// Backend turn primary key. pub turn_pk: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Agent version observed on this event. pub agent_version: Option, @@ -446,7 +459,7 @@ pub struct UsageEventListItem { pub session_id: String, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -518,7 +531,7 @@ pub struct UsageEventResponse { pub session_pk: Uuid, /// Backend turn primary key. pub turn_pk: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Agent version observed on this event. pub agent_version: Option, @@ -526,7 +539,7 @@ pub struct UsageEventResponse { pub session_id: String, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -550,6 +563,15 @@ pub struct UsageEventResponse { pub total_tokens: i64, /// Collector observation timestamp. pub observed_at: DateTime, + /// Agent-declared thread of the exchange. + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + #[serde(skip_serializing_if = "Option::is_none")] + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, /// Collector metadata retained without schema expansion. pub metadata: Value, /// Ordered attachment metadata associated with the event. @@ -588,7 +610,7 @@ pub struct UsageEventOutline { pub turn_pk: Uuid, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -606,8 +628,6 @@ pub struct UsageEventOutline { pub reasoning_tokens: i64, /// Provider-reported or derived total-token count. pub total_tokens: i64, - /// Whether the collector estimated token usage. - pub token_usage_estimated: bool, /// Collector observation timestamp. pub observed_at: DateTime, /// Text length in characters; zero when text is absent. @@ -617,26 +637,6 @@ pub struct UsageEventOutline { pub preview: Option, /// Number of image attachments on the event. pub attachment_count: i64, - /// Provider response-chain identifiers needed by timeline ordering. - #[serde(skip)] - pub ordering_metadata: Value, -} - -impl UsageEventOutline { - /// Builds the minimal metadata object consumed by timeline ordering. - pub fn ordering_metadata( - response_id: Option, - previous_response_id: Option, - ) -> Value { - let mut metadata = serde_json::Map::new(); - if let Some(value) = response_id { - metadata.insert("response_id".to_owned(), Value::String(value)); - } - if let Some(value) = previous_response_id { - metadata.insert("previous_response_id".to_owned(), Value::String(value)); - } - Value::Object(metadata) - } } #[derive(Debug, Serialize)] @@ -648,7 +648,7 @@ pub struct SessionInfo { pub user_id: Uuid, /// Most recently associated device context. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently known Agent version. pub agent_version: Option, @@ -669,25 +669,6 @@ pub struct SessionInfo { pub parent_session_pk: Option, } -/// Extracts the endpoint working directory retained in session metadata. -#[must_use] -pub fn session_working_directory(metadata: &Value) -> Option { - metadata - .get(WORKING_DIRECTORY_METADATA_KEY) - .and_then(Value::as_str) - .filter(|value| !value.is_empty() && value.len() <= 4_096) - .map(str::to_owned) -} - -/// Keeps only stable session-level metadata from an event. -#[must_use] -pub fn session_metadata_from_event(metadata: &Value) -> Value { - session_working_directory(metadata).map_or_else( - || json!({}), - |working_directory| json!({WORKING_DIRECTORY_METADATA_KEY: working_directory}), - ) -} - #[derive(Debug, Serialize)] /// One normalized turn and its deterministically ordered events. pub struct TurnTimeline { @@ -713,7 +694,6 @@ mod tests { use super::{ IngestEventsRequest, RawEventFields, RawEventsQuery, SummaryFields, SummaryQuery, SummaryResponse, SummaryRow, TokenUsageSummaryResponse, UsageEventType, - session_metadata_from_event, session_working_directory, }; use serde_json::json; @@ -741,10 +721,7 @@ mod tests { }, "event_type": "request", "text": "Explain this repository.", - "token_usage": { - "input_tokens": 10_i64, - "cache_read_tokens": 2_i64 - }, + "token_usage": {"input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 2_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 12_i64}, "metadata": { "source": "unit-test" } @@ -787,14 +764,13 @@ mod tests { "event_id": "evt-usage-002", "observed_at": "2026-05-20T01:03:03Z", "device": { - "hostname": "alice-mbp", + "host_name": "alice-mbp", "platform": "macos" }, "agent": { - "name": "Claude Code" + "name": "claude-code" }, "session_id": "session-002", - "turn_index": 2_i32, "llm": { "provider": "Anthropic", "model": "claude-4" @@ -816,7 +792,12 @@ mod tests { ); assert_eq!( event.token_usage.total_tokens, None, - "omitted token_usage should default to empty token counters" + "omitted token_usage should leave every counter absent" + ); + assert_eq!(event.turn_id, None, "an omitted turn names no turn"); + assert!( + !event.naming_thread, + "an omitted flag is not a naming thread" ); assert_eq!( event.metadata, @@ -825,23 +806,6 @@ mod tests { ); } - #[test] - fn session_metadata_keeps_only_a_bounded_working_directory() { - let metadata = json!({ - "working_directory": "/Users/alice/repo", - "request_hash": "sensitive-event-detail" - }); - - assert_eq!( - session_working_directory(&metadata).as_deref(), - Some("/Users/alice/repo") - ); - assert_eq!( - session_metadata_from_event(&metadata), - json!({"working_directory": "/Users/alice/repo"}) - ); - } - #[test] fn summary_query_accepts_token_usage_fields_mode() { let query = serde_json::from_value::(json!({ diff --git a/crates/abyss-backend/src/usage/persistence.rs b/crates/abyss-backend/src/usage/persistence.rs index cdfd0fc..e8938f8 100644 --- a/crates/abyss-backend/src/usage/persistence.rs +++ b/crates/abyss-backend/src/usage/persistence.rs @@ -9,7 +9,7 @@ use sha2::{Digest, Sha256}; use crate::error::AppError; use super::{ - IngestEventsRequest, IngestUsageEvent, + IngestEventsRequest, attachments::{ValidatedImageAttachment, validate_image_attachments}, }; @@ -27,7 +27,6 @@ pub fn validate_batch( // its table locks are acquired by either storage implementation. let mut attachments = Vec::with_capacity(request.events.len()); for event in &request.events { - validate_event(event)?; attachments.push(validate_image_attachments(&event.attachments)?); } if request.diagnostic_captures.len() > max_batch_size { @@ -52,88 +51,6 @@ pub fn validate_batch( Ok(attachments) } -pub fn validate_event(event: &IngestUsageEvent) -> Result<(), AppError> { - require_non_empty(&event.event_id, "event_id")?; - require_non_empty(&event.device.host_name, "device.host_name")?; - require_non_empty(&event.device.platform, "device.platform")?; - require_non_empty(&event.agent.name, "agent.name")?; - require_non_empty(&event.session_id, "session_id")?; - require_non_empty(&event.llm.provider, "llm.provider")?; - require_non_empty(&event.llm.model, "llm.model")?; - let tokens = normalized_tokens(event)?; - if tokens.total == 0 { - return Ok(()); - } - if tokens.total < tokens.component_total() { - tracing::debug!( - event_id = %event.event_id, - "provider total_tokens is lower than visible token components" - ); - } - Ok(()) -} - -fn require_non_empty(value: &str, field: &str) -> Result<(), AppError> { - if value.trim().is_empty() { - return Err(AppError::validation(format!("{field} is required"))); - } - Ok(()) -} - -pub struct NormalizedTokens { - pub input: i64, - pub output: i64, - pub cache_read: i64, - pub cache_write: i64, - pub reasoning: i64, - pub total: i64, -} - -impl NormalizedTokens { - /// Sum of the disjoint input, cache read, cache write, and output counters. - /// - /// Reasoning is a breakdown of output and is never added again. - pub const fn component_total(&self) -> i64 { - self.input - .saturating_add(self.cache_read) - .saturating_add(self.cache_write) - .saturating_add(self.output) - } -} - -pub fn normalized_tokens(event: &IngestUsageEvent) -> Result { - let input = normalize_token(event.token_usage.input_tokens, "input_tokens")?; - let output = normalize_token(event.token_usage.output_tokens, "output_tokens")?; - let cache_read = normalize_token(event.token_usage.cache_read_tokens, "cache_read_tokens")?; - let cache_write = normalize_token(event.token_usage.cache_write_tokens, "cache_write_tokens")?; - let reasoning = normalize_token(event.token_usage.reasoning_tokens, "reasoning_tokens")?; - let explicit_total = event - .token_usage - .total_tokens - .map(|value| normalize_token(Some(value), "total_tokens")) - .transpose()?; - let mut tokens = NormalizedTokens { - input, - output, - cache_read, - cache_write, - reasoning, - total: 0, - }; - // Preserve an explicit producer total; derive it from the disjoint - // counters only when the field is absent. - tokens.total = explicit_total.unwrap_or_else(|| tokens.component_total()); - Ok(tokens) -} - -fn normalize_token(value: Option, field: &str) -> Result { - let value = value.unwrap_or(0); - if value < 0 { - return Err(AppError::validation(format!("{field} must be >= 0"))); - } - Ok(value) -} - pub fn validate_event_type(value: &str) -> Result { match value.trim().to_ascii_lowercase().as_str() { "request" => Ok("request".to_owned()), @@ -176,61 +93,8 @@ pub fn parse_group_by(raw: Option<&str>) -> Vec { } } -pub fn normalize_slug(value: &str) -> String { - value - .trim() - .to_ascii_lowercase() - .chars() - .map(|character| match character { - '_' | ' ' => '-', - other => other, - }) - .collect() -} - -pub fn canonical_agent_name(value: &str) -> String { - let normalized = normalize_slug(value); - match normalized.as_str() { - "claude" | "claude-desktop" => "claude-code".to_owned(), - _ => normalized, - } -} - -pub fn agent_name_filter_values(value: &str) -> Vec { - let canonical = canonical_agent_name(value); - if canonical == "claude-code" { - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] - } else { - vec![canonical] - } -} - -/// Stored Agent names matching any requested Agent; empty matches every Agent. -pub fn agent_names_filter(values: &[String]) -> Vec { - distinct( - values - .iter() - .filter_map(|value| non_empty(value)) - .flat_map(agent_name_filter_values), - ) -} - -/// Provider slugs matching any requested provider; empty matches every provider. -pub fn providers_filter(values: &[String]) -> Vec { - distinct( - values - .iter() - .filter_map(|value| non_empty(value)) - .map(normalize_slug), - ) -} - -/// Trimmed values compared exactly, such as models or session identifiers; -/// empty matches every value. +/// Trimmed values compared exactly, such as Agent names, providers, models, or +/// session identifiers; empty matches every value. pub fn exact_values_filter(values: &[String]) -> Vec { distinct( values @@ -251,13 +115,6 @@ pub fn normalize_text(value: &str) -> String { value.trim().to_owned() } -pub fn normalized_optional(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - pub fn non_empty(value: &str) -> Option<&str> { (!value.trim().is_empty()).then_some(value) } @@ -292,146 +149,24 @@ pub fn sha256_bytes(value: &str) -> Vec { Sha256::digest(value.as_bytes()).to_vec() } -pub struct TurnIdentity { - pub kind: TurnIdentityKind, - pub value: String, -} - -#[derive(Clone, Copy)] -pub enum TurnIdentityKind { - CodexTurnId, - ClaudeTurnId, - TurnId, - ResponseId, - MessageId, - RequestHash, -} - -impl TurnIdentityKind { - pub const fn metadata_key(self) -> &'static str { - match self { - Self::CodexTurnId => "codex_turn_id", - Self::ClaudeTurnId => "claude_turn_id", - Self::TurnId => "turn_id", - Self::ResponseId => "response_id", - Self::MessageId => "message_id", - Self::RequestHash => "request_hash", - } - } -} - -pub fn turn_identity(event: &IngestUsageEvent) -> Option { - turn_identity_from_metadata(&event.metadata) -} - -pub fn turn_identity_from_metadata(metadata: &serde_json::Value) -> Option { - // Prefer Agent-level identities because one user turn may contain several - // provider round trips. - [ - TurnIdentityKind::CodexTurnId, - TurnIdentityKind::ClaudeTurnId, - TurnIdentityKind::TurnId, - TurnIdentityKind::ResponseId, - TurnIdentityKind::MessageId, - TurnIdentityKind::RequestHash, - ] - .into_iter() - .find_map(|kind| { - metadata_string(metadata, kind.metadata_key()).map(|value| TurnIdentity { kind, value }) - }) -} - -/// A Claude Code event outside the main loop: a side request (recap, title, -/// suggestion, compaction, ...) or a subagent request. It carries no -/// `claude_turn_id` and belongs to the session turn it occurs in. -pub struct SessionTurnAttachment { - /// `cc_prompt_id` of the Claude Code billing header, shared with the - /// main-loop requests of the prompt a subagent works for. - pub prompt_id: Option, -} - -/// Billing-header field naming the user prompt a Claude Code request serves. -const CLAUDE_CODE_PROMPT_ID_KEY: &str = "cc_prompt_id"; -const CLAUDE_CODE_BILLING_KEY: &str = "claude_code_billing"; - -/// Storage resolves an attachment in order: the turn of a main-loop event with -/// the same prompt id, the latest turn started at or before the event, the -/// earliest turn, and only for a session without turns a turn of its own. -/// Other Agents and Claude Code main-loop events keep their turn identity. -pub fn session_turn_attachment( - agent_name: &str, - metadata: &serde_json::Value, -) -> Option { - if agent_name != "claude-code" - || metadata_string(metadata, TurnIdentityKind::ClaudeTurnId.metadata_key()).is_some() - { - return None; - } - Some(SessionTurnAttachment { - prompt_id: metadata - .get(CLAUDE_CODE_BILLING_KEY) - .and_then(|billing| metadata_string(billing, CLAUDE_CODE_PROMPT_ID_KEY)), - }) -} - -fn metadata_string(metadata: &serde_json::Value, key: &str) -> Option { - metadata - .get(key) - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - #[cfg(test)] mod tests { - use serde_json::json; - - use super::{ - TurnIdentityKind, canonical_agent_name, escape_like_pattern, parse_group_by, - session_turn_attachment, turn_identity_from_metadata, - }; + use super::{escape_like_pattern, exact_values_filter, parse_group_by}; #[test] fn shared_normalization_is_backend_independent() { - assert_eq!(canonical_agent_name(" Claude Desktop "), "claude-code"); + // Agent names and providers are matched exactly as reported: Claude + // Desktop stays apart from Claude Code and case is not folded. + assert_eq!( + exact_values_filter(&[ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "Codex".to_owned(), + "claude-code".to_owned(), + ]), + ["Codex", "claude-code", "claude-desktop"] + ); assert_eq!(parse_group_by(Some("day,llm_model")), ["day", "model"]); assert_eq!(escape_like_pattern(r"a%b_c\d"), r"a\%b\_c\\d"); } - - #[test] - fn agent_turn_identity_wins_over_provider_identities() { - let identity = turn_identity_from_metadata(&json!({ - "response_id": "response", - "codex_turn_id": "turn" - })) - .expect("turn identity should be extracted"); - assert!(matches!(identity.kind, TurnIdentityKind::CodexTurnId)); - assert_eq!(identity.value, "turn"); - let identity = turn_identity_from_metadata(&json!({ - "response_id": "response", - "turn_id": "resp_1" - })) - .expect("turn identity should be extracted"); - assert!(matches!(identity.kind, TurnIdentityKind::TurnId)); - } - - #[test] - fn only_claude_code_events_without_turn_id_attach_to_a_session_turn() { - let billing = json!({"claude_code_billing": {"cc_prompt_id": " prompt "}}); - let attachment = session_turn_attachment("claude-code", &billing) - .expect("a Claude Code side request should attach"); - assert_eq!(attachment.prompt_id.as_deref(), Some("prompt")); - let side = session_turn_attachment("claude-code", &json!({"message_id": "msg"})) - .expect("a request without billing header should attach"); - assert_eq!(side.prompt_id, None); - assert!( - session_turn_attachment( - "claude-code", - &json!({"claude_turn_id": "cc_turn:3", "claude_code_billing": {"cc_prompt_id": "p"}}) - ) - .is_none() - ); - assert!(session_turn_attachment("codex", &billing).is_none()); - } } diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs index 5a9e1aa..bd931fb 100644 --- a/crates/abyss-backend/src/usage/session_list.rs +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -4,10 +4,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::{ - error::AppError, - usage::persistence::{agent_names_filter, exact_values_filter, providers_filter}, -}; +use crate::{error::AppError, usage::persistence::exact_values_filter}; const DEFAULT_PAGE_SIZE: u32 = 20; const MAX_PAGE_SIZE: u32 = 100; @@ -19,8 +16,8 @@ pub struct SessionListQuery { pub from: Option>, /// Keeps sessions that started before this instant. pub to: Option>, - /// Agent-name filter; repeat the key for several Agents. Claude aliases - /// match together. + /// Agent-name filter, matched exactly as reported; repeat the key for + /// several Agents. #[serde(default)] pub agent_name: Vec, /// Keeps sessions with at least one event from any of these LLM providers. @@ -42,7 +39,7 @@ pub struct ValidatedSessionListQuery { pub to: Option>, /// Stored agent names to match, or empty for every Agent. pub agent_names: Vec, - /// Normalized provider slugs; a session must have used one, or empty for any. + /// Exact providers; a session must have used one, or empty for any. pub llm_providers: Vec, /// Exact models; a session must have used one, or empty for any. pub llm_models: Vec, @@ -74,8 +71,8 @@ impl SessionListQuery { Ok(ValidatedSessionListQuery { from: self.from, to: self.to, - agent_names: agent_names_filter(&self.agent_name), - llm_providers: providers_filter(&self.llm_provider), + agent_names: exact_values_filter(&self.agent_name), + llm_providers: exact_values_filter(&self.llm_provider), llm_models: exact_values_filter(&self.llm_model), page, page_size, @@ -151,9 +148,13 @@ mod tests { } #[test] - fn expands_claude_aliases_and_computes_the_offset() { + fn keeps_agent_names_exact_and_computes_the_offset() { let query = SessionListQuery { - agent_name: vec![" Claude ".to_owned(), "codex".to_owned()], + agent_name: vec![ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "codex".to_owned(), + ], page: Some(3), page_size: Some(25), ..SessionListQuery::default() @@ -163,12 +164,12 @@ mod tests { assert_eq!(query.offset, 50); assert_eq!( query.agent_names, - ["claude", "claude-code", "claude-desktop", "codex"] + ["claude-code", "claude-desktop", "codex"] ); } #[test] - fn normalizes_providers_and_keeps_models_verbatim() { + fn keeps_providers_and_models_verbatim() { let query = SessionListQuery { llm_provider: vec![" OpenAI ".to_owned(), "anthropic".to_owned()], llm_model: vec![" gpt-5 ".to_owned(), "Claude-Opus".to_owned()], @@ -176,7 +177,7 @@ mod tests { } .validate() .expect("valid"); - assert_eq!(query.llm_providers, ["anthropic", "openai"]); + assert_eq!(query.llm_providers, ["OpenAI", "anthropic"]); assert_eq!(query.llm_models, ["Claude-Opus", "gpt-5"]); let blank = SessionListQuery { llm_model: vec![" ".to_owned()], From 6b9b6f6ea0d8d19146e4d9dd7f366d9dab85cae0 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:57 -0700 Subject: [PATCH 28/30] feat(search)!: index tool input as written Tool inputs were parsed as JSON to pull `command` / `cmd` / `file` / `file_path` / `path` values, plus the session working directory, into separate commands and file paths fields. The backend no longer guesses tool argument names: the search projection, the SQLite FTS snippets, and the Elasticsearch document, mapping, query fields, and highlights drop both fields. Tool input stays searchable through `tool_content`. Search results report the Agent name as stored, and test fixtures carry the new exchange identity fields and explicit token counters. Co-Authored-By: Claude Opus 5.5 --- crates/abyss-backend/src/search/projection.rs | 46 ------------------- .../src/storage/postgres/search/document.rs | 24 ++++------ .../storage/postgres/search/elasticsearch.rs | 26 +++-------- .../src/storage/sqlite/search.rs | 15 +----- 4 files changed, 19 insertions(+), 92 deletions(-) diff --git a/crates/abyss-backend/src/search/projection.rs b/crates/abyss-backend/src/search/projection.rs index 4bba669..c4e4733 100644 --- a/crates/abyss-backend/src/search/projection.rs +++ b/crates/abyss-backend/src/search/projection.rs @@ -7,15 +7,12 @@ use super::{HIGHLIGHT_END, HIGHLIGHT_START}; pub const MAX_CONTENT_CHARACTERS: usize = 1_000_000; const MAX_METADATA_FIELD_CHARACTERS: usize = 100_000; const MAX_EXTRACTED_VALUES: usize = 64; -const MAX_JSON_DEPTH: usize = 8; /// Searchable values derived from one source usage event. pub struct SearchProjection { pub content: Option, pub tool_names: Vec, pub tool_content: Vec, - pub commands: Vec, - pub file_paths: Vec, } impl SearchProjection { @@ -27,8 +24,6 @@ impl SearchProjection { .map(|value| truncate_characters(value, MAX_CONTENT_CHARACTERS)), tool_names: extracted.tool_names, tool_content: extracted.tool_content, - commands: extracted.commands, - file_paths: extracted.file_paths, } } } @@ -36,8 +31,6 @@ impl SearchProjection { struct SearchableMetadata { tool_names: Vec, tool_content: Vec, - commands: Vec, - file_paths: Vec, } impl SearchableMetadata { @@ -45,14 +38,8 @@ impl SearchableMetadata { let mut extracted = Self { tool_names: Vec::new(), tool_content: Vec::new(), - commands: Vec::new(), - file_paths: Vec::new(), }; - if let Some(working_directory) = metadata.get("working_directory").and_then(Value::as_str) { - extracted.push_file_path(working_directory); - } - // Ignore every other top-level key so headers, credentials, and image // data never enter a derived full-text index. let Some(segments) = metadata.get("content_segments").and_then(Value::as_array) else { @@ -69,9 +56,6 @@ impl SearchableMetadata { } if let Some(input) = object.get("input").and_then(Value::as_str) { push_bounded(&mut extracted.tool_content, input); - if let Ok(value) = serde_json::from_str::(input) { - extracted.extract_structured_input(&value, MAX_JSON_DEPTH); - } } } Some("tool_result") => { @@ -84,36 +68,6 @@ impl SearchableMetadata { } extracted } - - fn extract_structured_input(&mut self, value: &Value, remaining_depth: usize) { - if remaining_depth == 0 { - return; - } - match value { - Value::Object(object) => { - for (key, child) in object.iter().take(MAX_EXTRACTED_VALUES) { - if let Some(text) = child.as_str() { - match key.as_str() { - "command" | "cmd" => push_bounded(&mut self.commands, text), - "file" | "file_path" | "path" => self.push_file_path(text), - _ => {} - } - } - self.extract_structured_input(child, remaining_depth.saturating_sub(1)); - } - } - Value::Array(items) => { - for item in items.iter().take(MAX_EXTRACTED_VALUES) { - self.extract_structured_input(item, remaining_depth.saturating_sub(1)); - } - } - _ => {} - } - } - - fn push_file_path(&mut self, value: &str) { - push_bounded(&mut self.file_paths, value); - } } fn push_bounded(values: &mut Vec, value: &str) { diff --git a/crates/abyss-backend/src/storage/postgres/search/document.rs b/crates/abyss-backend/src/storage/postgres/search/document.rs index 0c0d7ec..1120039 100644 --- a/crates/abyss-backend/src/storage/postgres/search/document.rs +++ b/crates/abyss-backend/src/storage/postgres/search/document.rs @@ -31,9 +31,9 @@ pub struct SearchDocument { pub turn_pk: Uuid, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical Agent name used for exact filtering. + /// Agent name as reported by the collector, used for exact filtering. pub agent_name: String, - /// Canonical LLM provider slug used for exact filtering. + /// LLM provider as reported by the collector, used for exact filtering. pub llm_provider: String, /// Provider-specific model identifier used for exact filtering. pub llm_model: String, @@ -50,12 +50,6 @@ pub struct SearchDocument { /// Bounded raw tool inputs and outputs. #[serde(skip_serializing_if = "Vec::is_empty")] pub tool_content: Vec, - /// Bounded command strings extracted from structured tool input. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub commands: Vec, - /// Bounded paths and working directories extracted from allowlisted fields. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub file_paths: Vec, } impl SearchDocument { @@ -78,8 +72,6 @@ impl SearchDocument { content: projection.content, tool_names: projection.tool_names, tool_content: projection.tool_content, - commands: projection.commands, - file_paths: projection.file_paths, } } } @@ -98,7 +90,6 @@ mod tests { fn extracts_only_searchable_tool_metadata() { let mut event = usage_event(); event.metadata = json!({ - "working_directory": "/Users/alice/repo", "authorization": "Bearer secret", "content_segments": [ { @@ -120,8 +111,11 @@ mod tests { let document = SearchDocument::from_event(event); assert_eq!(document.tool_names, ["Bash"]); - assert_eq!(document.commands, ["cargo test"]); - assert_eq!(document.file_paths, ["/Users/alice/repo", "src/lib.rs"]); + // Tool inputs are searchable as written; no field is pulled out by name. + assert_eq!( + document.tool_content[0], + "{\"command\":\"cargo test\",\"nested\":{\"file_path\":\"src/lib.rs\"}}" + ); assert_eq!(document.tool_content.len(), 2); let serialized = serde_json::to_string(&document).expect("document should serialize"); assert!(!serialized.contains("Bearer secret")); @@ -153,7 +147,6 @@ mod tests { "before {HIGHLIGHT_START}match{HIGHLIGHT_END} after" )); event.metadata = json!({ - "working_directory": format!("/tmp/{HIGHLIGHT_START}repo"), "content_segments": [ { "type": "tool_call", @@ -204,6 +197,9 @@ mod tests { observed_at: Utc::now(), metadata: json!({}), created_at: Utc::now(), + stream_id: None, + response_id: None, + previous_response_id: None, } } } diff --git a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs index 578e328..8757c4b 100644 --- a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs +++ b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs @@ -246,7 +246,7 @@ pub struct MatchedEvent { pub turn_index: i32, /// Request or response side. pub event_type: String, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -300,9 +300,7 @@ fn index_definition() -> Value { "observed_at": {"type": "date"}, "content": {"type": "text"}, "tool_names": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, - "tool_content": {"type": "text"}, - "commands": {"type": "text"}, - "file_paths": {"type": "text"} + "tool_content": {"type": "text"} } } }) @@ -341,8 +339,6 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { "fields": [ "content^4", "session_id^3", - "commands^3", - "file_paths^3", "tool_names^2", "tool_content" ], @@ -375,8 +371,6 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { "number_of_fragments": 2, "fields": { "content": {}, - "commands": {}, - "file_paths": {}, "tool_names": {}, "tool_content": {} } @@ -603,17 +597,11 @@ impl TryFrom for SearchMatchPage { } fn ordered_fragments(highlights: &HashMap>) -> Vec { - [ - "content", - "commands", - "file_paths", - "tool_names", - "tool_content", - ] - .into_iter() - .flat_map(|field| highlights.get(field).into_iter().flatten().cloned()) - .take(2) - .collect() + ["content", "tool_names", "tool_content"] + .into_iter() + .flat_map(|field| highlights.get(field).into_iter().flatten().cloned()) + .take(2) + .collect() } #[cfg(test)] diff --git a/crates/abyss-backend/src/storage/sqlite/search.rs b/crates/abyss-backend/src/storage/sqlite/search.rs index 7f68cca..2bc5c85 100644 --- a/crates/abyss-backend/src/storage/sqlite/search.rs +++ b/crates/abyss-backend/src/storage/sqlite/search.rs @@ -18,7 +18,6 @@ use crate::{ HIGHLIGHT_END, HIGHLIGHT_START, SearchFragment, SessionSearchMatch, SessionSearchQuery, SessionSearchResponse, SessionSearchResult, ValidatedSearchQuery, }, - usage::persistence::canonical_agent_name, }; use super::{ @@ -75,7 +74,7 @@ fn search( Some(SessionSearchResult { session_pk, session_id: details.session_id, - agent_name: canonical_agent_name(&details.agent_name), + agent_name: details.agent_name, agent_version: details.agent_version, host_name: details.host_name, platform: details.platform, @@ -381,10 +380,6 @@ fn load_matches( top.observed_at AS observed_at, snippet(usage_events_fts, 4, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS content_fragment, - snippet(usage_events_fts, 7, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) - AS command_fragment, - snippet(usage_events_fts, 8, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) - AS path_fragment, snippet(usage_events_fts, 5, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS tool_name_fragment, snippet(usage_events_fts, 6, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) @@ -429,10 +424,6 @@ struct MatchedEventRow { #[diesel(sql_type = Nullable)] content_fragment: Option, #[diesel(sql_type = Nullable)] - command_fragment: Option, - #[diesel(sql_type = Nullable)] - path_fragment: Option, - #[diesel(sql_type = Nullable)] tool_name_fragment: Option, #[diesel(sql_type = Nullable)] tool_content_fragment: Option, @@ -442,8 +433,6 @@ impl MatchedEventRow { fn into_matched(self) -> Result { let fragments = [ self.content_fragment, - self.command_fragment, - self.path_fragment, self.tool_name_fragment, self.tool_content_fragment, ] @@ -615,7 +604,7 @@ mod tests { "llm": {"provider": "openai", "model": "gpt-5"}, "event_type": "response", "text": text, - "token_usage": {"output_tokens": 1}, + "token_usage": {"input_tokens": 0_i64, "output_tokens": 1_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64}, "metadata": {} }) } From 5f82fd4fe581c5f4472840c8f67dbc8276ce1de5 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:57 -0700 Subject: [PATCH 29/30] feat(trajectory)!: place streams by declared keys and keep roles as reported Trajectories no longer interpret prompt text or provider-specific fields. - Streams: a request whose `previous_response_id` names a known response joins that response's stream, else a request with a `stream_id` joins that stream, else it opens a stream of its own; a response joins the stream of its exchange by `response_id`. The system-prompt hash, `claude_thread`, latest-request inference, and the billing header / `` stripping are removed. - Continuations and responses show every message they carry; only full requests are compared with the stream history. - Roles are kept as reported. Items are `message` with the original `role`, or `text`, `tool_call`, `tool_result`, and `compaction`. The opening instructions are the first message when `instructions_present` is set and show only when they change; messages without text are left out. - Compaction is recognized from `purpose == "compaction"` on the exchange instead of Codex prompt text, and its response is matched by `response_id`. - Conversations require `completeness`; the legacy-collector branches that fell back to flattened text are removed. - `token_usage_available` means any counter is non-zero, and `token_usage_estimated` is gone. - Session titles mark naming threads from `naming_thread` instead of Codex `thread_source`, and take the first `user` message as default. Co-Authored-By: Claude Opus 5.5 --- .../abyss-backend/src/usage/conversation.rs | 101 +- .../abyss-backend/src/usage/session_title.rs | 40 +- crates/abyss-backend/src/usage/trajectory.rs | 900 +++++++++--------- .../src/usage/trajectory/conversation.rs | 299 +++--- 4 files changed, 633 insertions(+), 707 deletions(-) diff --git a/crates/abyss-backend/src/usage/conversation.rs b/crates/abyss-backend/src/usage/conversation.rs index 9b9a5a9..9814625 100644 --- a/crates/abyss-backend/src/usage/conversation.rs +++ b/crates/abyss-backend/src/usage/conversation.rs @@ -6,12 +6,11 @@ use serde_json::Value; #[derive(Deserialize)] pub struct Conversation { pub messages: Vec, - #[serde(default)] - pub previous_response_id: Option, + /// Whether the first message is the request's own instructions, such as + /// a system prompt or OpenAI `instructions`; absent means none. #[serde(default)] pub instructions_present: bool, - #[serde(default)] - completeness: Option, + completeness: Completeness, } #[derive(Deserialize, PartialEq)] @@ -22,68 +21,22 @@ enum Completeness { Unsupported, } -/// Message roles the collector emits. Roles outside this set, including the -/// placeholder `request` from older collectors, read as [`Role::Unknown`]. -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)] -#[serde(rename_all = "snake_case")] -pub enum Role { - System, - Developer, - User, - Assistant, - Reasoning, - Tool, - #[serde(other)] - Unknown, -} - -impl Role { - pub const fn as_str(self) -> &'static str { - match self { - Self::System => "system", - Self::Developer => "developer", - Self::User => "user", - Self::Assistant => "assistant", - Self::Reasoning => "reasoning", - Self::Tool => "tool", - Self::Unknown => "unknown", - } - } -} - +/// One message as the collector reports it; its role is kept as reported. #[derive(Clone, Deserialize)] pub struct Message { #[serde(default)] pub id: Option, - pub role: Role, + pub role: String, pub text: String, } impl Conversation { - /// Whether the event carries a conversation from a collector that reports - /// completeness. Older collectors stored messages without it; their - /// requests keep using the flattened text where they did before. - pub fn declared(metadata: &Value) -> bool { - metadata.pointer("/conversation/completeness").is_some() - } - - /// Thread source the Agent declared for the request, read without - /// decoding its messages. - pub fn thread_source(metadata: &Value) -> Option<&str> { - metadata.get("conversation")?.get("thread_source")?.as_str() - } - + /// The completely decoded conversation of an event, as the `AgentEvent` + /// contract defines it. pub fn read(metadata: &Value) -> Option { let value = metadata.get("conversation")?; let conversation: Self = Self::deserialize(value).ok()?; - if conversation - .completeness - .as_ref() - .is_some_and(|state| *state != Completeness::Complete) - { - return None; - } - Some(conversation) + (conversation.completeness == Completeness::Complete).then_some(conversation) } } @@ -92,20 +45,40 @@ mod tests { use super::*; #[test] - fn legacy_conversations_read_with_unknown_roles() { - let metadata = serde_json::json!({"conversation": {"messages": [ - {"role": "request", "text": "placeholder"}, - {"role": "function", "text": "legacy"}, - {"role": "user", "text": "hello"} - ]}}); - let conversation = Conversation::read(&metadata).expect("legacy shape"); + fn roles_are_kept_as_reported() { + let metadata = serde_json::json!({"conversation": { + "messages": [ + {"role": "reasoning", "text": "plan"}, + {"role": "some_new_role", "text": "other"}, + {"role": "user", "text": "hello"} + ], + "instructions_present": false, + "completeness": "complete" + }}); + let conversation = Conversation::read(&metadata).expect("complete conversation"); assert_eq!( conversation .messages .iter() - .map(|message| message.role) + .map(|message| message.role.as_str()) .collect::>(), - [Role::Unknown, Role::Unknown, Role::User] + ["reasoning", "some_new_role", "user"] ); } + + #[test] + fn conversations_outside_the_contract_are_not_read() { + let messages = serde_json::json!([{"role": "user", "text": "hello"}]); + for conversation in [ + serde_json::json!({"messages": messages, "instructions_present": false}), + serde_json::json!({ + "messages": messages, + "instructions_present": false, + "completeness": "partial" + }), + ] { + let metadata = serde_json::json!({"conversation": conversation}); + assert!(Conversation::read(&metadata).is_none()); + } + } } diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs index 6cf38e5..733062f 100644 --- a/crates/abyss-backend/src/usage/session_title.rs +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -5,18 +5,17 @@ //! `purpose`, and the response carries the `session_title` it produced for its //! own session. Titles are therefore read without interpreting message text. //! -//! Codex names a task from a separate thread whose requests do not say which -//! session they name, so its task sessions keep their first user input and the -//! naming threads are only marked, from the protocol's `thread_source`. +//! An Agent may instead name a task from a separate thread whose requests do +//! not say which session they name. The collector marks those requests with +//! `naming_thread`; their sessions are only marked, and the task sessions keep +//! their first user input. use serde_json::Value; -use super::conversation::{Conversation, Role}; +use super::conversation::Conversation; /// Session purpose stored for threads that name another session. pub const TITLE_GENERATION_PURPOSE: &str = "title_generation"; -/// Thread source Codex declares on its naming threads. -const TITLE_THREAD_SOURCE: &str = "thread_title"; /// Longest default title, in characters. const DEFAULT_TITLE_CHARS: usize = 80; @@ -35,14 +34,14 @@ impl<'a> TitleContribution<'a> { /// Classifies an event from collector fields without reading its messages. /// Requests the Agent made for its own purposes, such as naming calls, are /// not user input. - pub fn of(event_type: &str, metadata: &'a Value) -> Option { + pub fn of(event_type: &str, metadata: &'a Value, naming_thread: bool) -> Option { if let Some(title) = metadata.get("session_title").and_then(Value::as_str) { return Some(Self::Agent(title)); } if event_type != "request" || metadata.get("purpose").is_some() { return None; } - if Conversation::thread_source(metadata) == Some(TITLE_THREAD_SOURCE) { + if naming_thread { return Some(Self::TitleThread); } Some(Self::UserInput) @@ -54,7 +53,7 @@ pub fn default_title(metadata: &Value) -> Option { let input = Conversation::read(metadata)? .messages .into_iter() - .find(|message| message.role == Role::User && !message.text.trim().is_empty())? + .find(|message| message.role == "user" && !message.text.trim().is_empty())? .text; let input = input.split_whitespace().collect::>().join(" "); let mut chars = input.chars(); @@ -133,7 +132,8 @@ mod tests { assert_eq!( TitleContribution::of( "response", - &json!({"purpose":"session_title","session_title":"登录修复"}) + &json!({"purpose":"session_title","session_title":"登录修复"}), + false ), Some(TitleContribution::Agent("登录修复")) ); @@ -143,7 +143,8 @@ mod tests { "response", &request(&json!({"messages":[ {"role":"assistant","text":"{\"title\":\"Quoted\"}"} - ]})) + ]})), + false ), None ); @@ -153,24 +154,19 @@ mod tests { fn naming_requests_are_not_user_input() { let mut metadata = request(&json!({"messages":[{"role":"user","text":"…"}]})); metadata["purpose"] = json!("session_title"); - assert_eq!(TitleContribution::of("request", &metadata), None); + assert_eq!(TitleContribution::of("request", &metadata, false), None); } #[test] - fn codex_naming_threads_are_marked_from_their_thread_source() { - let messages = json!([{"role":"user","text":"Generate a concise title"}]); + fn naming_threads_are_marked_from_the_collector_flag() { + let metadata = + request(&json!({"messages":[{"role":"user","text":"Generate a concise title"}]})); assert_eq!( - TitleContribution::of( - "request", - &request(&json!({"messages":messages,"thread_source":"thread_title"})) - ), + TitleContribution::of("request", &metadata, true), Some(TitleContribution::TitleThread) ); assert_eq!( - TitleContribution::of( - "request", - &request(&json!({"messages":messages,"thread_source":"user"})) - ), + TitleContribution::of("request", &metadata, false), Some(TitleContribution::UserInput) ); } diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs index 5d72b9a..9329c9a 100644 --- a/crates/abyss-backend/src/usage/trajectory.rs +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -7,7 +7,7 @@ mod conversation; -use conversation::ConversationProjection; +use conversation::{ConversationProjection, Placement}; use std::{ collections::{HashMap, HashSet, hash_map::DefaultHasher}, @@ -21,15 +21,9 @@ use uuid::Uuid; use super::{SessionInfo, SessionTimelineResponse, UsageEventResponse}; -/// Collectors join request fragments with this separator. -const PARAGRAPH_SEPARATOR: &str = "\n\n"; -/// Codex opens the request that asks the model for a context handoff summary -/// with this prompt when it compacts its context. -const COMPACTION_PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION."; -/// Codex opens the compacted history it sends afterwards with this sentence, -/// followed by the handoff summary. -const COMPACTION_HANDOFF: &str = "Another language model started to solve this problem and \ - produced a summary of its thinking process."; +/// Purpose the collector declares on both events of a call that asks the +/// model to summarize the conversation so far. +const COMPACTION_PURPOSE: &str = "compaction"; #[derive(Debug, Serialize)] /// Session information and its merged, de-duplicated trajectory items. @@ -67,20 +61,12 @@ pub struct TrajectoryEvent { pub reasoning_tokens: i64, /// Authoritative total for this event, counted once. pub total_tokens: i64, - /// Whether counters were reported or estimated rather than absent or hidden. + /// Whether any token counter is non-zero. pub token_usage_available: bool, - /// Whether the collector estimated token usage. - pub token_usage_estimated: bool, } impl From<&UsageEventResponse> for TrajectoryEvent { fn from(event: &UsageEventResponse) -> Self { - let source = event - .metadata - .get("token_usage_source") - .and_then(Value::as_str); - let hidden = - event.metadata.pointer("/content_policy/token_usage") == Some(&Value::Bool(false)); let has_usage = event.input_tokens != 0 || event.output_tokens != 0 || event.cache_read_tokens != 0 @@ -99,11 +85,7 @@ impl From<&UsageEventResponse> for TrajectoryEvent { cache_write_tokens: event.cache_write_tokens, reasoning_tokens: event.reasoning_tokens, total_tokens: event.total_tokens, - token_usage_available: !hidden - && source != Some("absent") - && (has_usage || source.is_some()), - token_usage_estimated: event.metadata.get("token_usage_estimated") - == Some(&Value::Bool(true)), + token_usage_available: has_usage, } } } @@ -112,20 +94,17 @@ impl From<&UsageEventResponse> for TrajectoryEvent { #[serde(rename_all = "snake_case")] /// Kind of one trajectory item. pub enum TrajectoryItemKind { - /// Request content whose message role is unavailable. - Request, - /// Explicit system or developer message. - Context, - /// Explicit user message not seen earlier in the session. - User, - /// Response text. - Assistant, + /// A conversation message not seen earlier in its stream, with the role + /// the collector reported. + Message, + /// Event text of an event whose conversation could not be read. + Text, /// Response tool call, with its result once one arrives. ToolCall, /// Request tool result whose call was not observed. ToolResult, - /// Context compaction: the summary prompt as `input` and the handoff - /// summary that replaced the earlier history as `output`. + /// Context compaction: the new user input of the compaction request as + /// `input` and the summary its response returned as `output`. Compaction, } @@ -134,8 +113,10 @@ pub enum TrajectoryItemKind { pub struct TrajectoryItem { /// Item kind. pub kind: TrajectoryItemKind, - /// Zero-based request stream, keyed by system prompt. Side requests such - /// as classifiers or subagents form their own streams. + /// Role of a message item, exactly as the collector reported it. + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Zero-based stream, placed by the identities the collector declares. pub stream: usize, /// Backend-normalized turn number. pub turn_index: i32, @@ -145,7 +126,7 @@ pub struct TrajectoryItem { pub observed_at: DateTime, /// Model of that event. pub llm_model: String, - /// Text of context, user, and assistant items. + /// Text of message and text items. #[serde(skip_serializing_if = "Option::is_none")] pub text: Option, /// Whether `text` is only the new tail of a paragraph that grew. @@ -172,6 +153,7 @@ impl TrajectoryItem { fn new(kind: TrajectoryItemKind, stream: usize, event: &UsageEventResponse) -> Self { Self { kind, + role: None, stream, turn_index: event.turn_index, event_pk: event.id, @@ -186,25 +168,6 @@ impl TrajectoryItem { output_event_pk: None, } } - - /// An empty item of another kind contributed by the same event. - fn sibling(&self, kind: TrajectoryItemKind) -> Self { - Self { - kind, - stream: self.stream, - turn_index: self.turn_index, - event_pk: self.event_pk, - observed_at: self.observed_at, - llm_model: self.llm_model.clone(), - text: None, - continued: false, - name: None, - call_id: None, - input: None, - output: None, - output_event_pk: None, - } - } } /// Merges an ordered session timeline into its trajectory. @@ -233,45 +196,37 @@ struct Merger { conversation: ConversationProjection, items: Vec, seen_results: HashSet, - streams: HashMap, /// Tool call item index by call identifier. calls: HashMap, - /// Stream of the most recent request; responses belong to it. - current_stream: usize, - /// Compaction item whose summary the next response carries. - pending_compaction: Option, - /// Handoff summaries already shown, so their replay is not shown again. - summaries: Vec, + /// Compaction item index by the response id of its exchange, until the + /// response carrying the summary arrives. + compactions: HashMap, } impl Merger { fn request(&mut self, event: &UsageEventResponse) { - self.pending_compaction = None; let first_new = self.items.len(); - if let Some(items) = - self.conversation - .project(event, &mut self.current_stream, &mut self.streams) - { + let placement = self.conversation.place(event); + if let Some(items) = self.conversation.project(event, placement) { self.items.extend(items); } else { - self.unclassified_request(event); + self.unclassified_request(event, placement); + } + if is_compaction(event) { + self.compaction(event, placement.stream, first_new); } - self.mark_compaction(first_new); - self.request_tools(event); + self.request_tools(event, placement.stream); } - fn unclassified_request(&mut self, event: &UsageEventResponse) { + fn unclassified_request(&mut self, event: &UsageEventResponse, placement: Placement) { if let Some(text) = event.text.as_ref().filter(|text| !text.is_empty()) { - let mut item = - TrajectoryItem::new(TrajectoryItemKind::Request, self.current_stream, event); + let mut item = TrajectoryItem::new(TrajectoryItemKind::Text, placement.stream, event); item.text = Some(text.clone()); self.items.push(item); } } - fn request_tools(&mut self, event: &UsageEventResponse) { - let stream = self.current_stream; - + fn request_tools(&mut self, event: &UsageEventResponse, stream: usize) { for segment in content_segments(&event.metadata) { if segment_str(segment, "type") != Some("tool_result") { continue; @@ -301,84 +256,50 @@ impl Merger { } } - /// Turns the compaction prompt into a compaction item and hides the - /// handoff summary when the compacted history replays it. - fn mark_compaction(&mut self, first_new: usize) { - for mut item in self.items.split_off(first_new) { - let Some(text) = item - .text - .clone() - .filter(|_| item.kind == TrajectoryItemKind::User) - else { + /// Merges the new user input of a compaction request into one compaction + /// item, which the response of the same exchange completes with its + /// summary. + fn compaction(&mut self, event: &UsageEventResponse, stream: usize, first_new: usize) { + let mut compaction = TrajectoryItem::new(TrajectoryItemKind::Compaction, stream, event); + let mut index = None; + let mut input = Vec::new(); + for item in self.items.split_off(first_new) { + if item.role.as_deref() != Some("user") { self.items.push(item); continue; - }; - // Legacy projections join all new paragraphs of a request, so the - // prompt may follow other new input in the same item. - if let Some(start) = paragraph_start(&text, COMPACTION_PROMPT) { - let (before, prompt) = text.split_at(start); - let mut compaction = item.sibling(TrajectoryItemKind::Compaction); - compaction.input = Some(prompt.trim().to_owned()); - let before = before.trim_end(); - if !before.is_empty() { - item.text = Some(before.to_owned()); - self.items.push(item); - } - self.pending_compaction = Some(self.items.len()); - self.items.push(compaction); - continue; - } - let Some(start) = paragraph_start(&text, COMPACTION_HANDOFF) else { - self.items.push(item); - continue; - }; - let (before, handoff) = text.split_at(start); - let before = before.trim_end(); - let replayed = self - .summaries - .iter() - .any(|summary| handoff.trim_end().ends_with(summary.as_str())); - // Without the summary request, the handoff is the only record of - // the compaction. - let recovered = (!replayed).then(|| { - let summary = handoff - .split_once('\n') - .map_or(handoff, |(_, rest)| rest) - .trim() - .to_owned(); - let mut compaction = item.sibling(TrajectoryItemKind::Compaction); - compaction.output = Some(summary.clone()); - compaction.output_event_pk = Some(item.event_pk); - self.summaries.push(summary); - compaction - }); - if !before.is_empty() { - item.text = Some(before.to_owned()); - self.items.push(item); } - self.items.extend(recovered); + index.get_or_insert(self.items.len()); + input.extend(item.text); + } + compaction.input = (!input.is_empty()).then(|| input.join("\n\n")); + let index = index.unwrap_or(self.items.len()); + self.items.insert(index, compaction); + if let Some(id) = &event.response_id { + self.compactions.insert(id.clone(), index); } } fn response(&mut self, event: &UsageEventResponse) { - let stream = self.current_stream; - let projected = - self.conversation - .project(event, &mut self.current_stream, &mut self.streams); - if let Some(index) = self.pending_compaction.take() { - // The response to the compaction prompt is the handoff summary. + let placement = self.conversation.place(event); + let stream = placement.stream; + let projected = self.conversation.project(event, placement); + let compaction = event + .response_id + .as_ref() + .and_then(|id| self.compactions.remove(id)); + if let Some(index) = compaction { + // The response of a compaction exchange is the summary. if let Some(summary) = event.text.as_deref().map(str::trim) && !summary.is_empty() && let Some(item) = self.items.get_mut(index) { item.output = Some(summary.to_owned()); item.output_event_pk = Some(event.id); - self.summaries.push(summary.to_owned()); } } else if let Some(items) = projected { self.items.extend(items); } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { - let mut item = TrajectoryItem::new(TrajectoryItemKind::Assistant, stream, event); + let mut item = TrajectoryItem::new(TrajectoryItemKind::Text, stream, event); item.text = Some(text.to_owned()); self.items.push(item); } @@ -398,16 +319,8 @@ impl Merger { } } -/// Byte offset of the paragraph that contains `marker`. -fn paragraph_start(text: &str, marker: &str) -> Option { - let position = text.find(marker)?; - Some( - text[..position] - .rfind(PARAGRAPH_SEPARATOR) - .map_or(0, |separator| { - separator.saturating_add(PARAGRAPH_SEPARATOR.len()) - }), - ) +fn is_compaction(event: &UsageEventResponse) -> bool { + event.metadata.get("purpose").and_then(Value::as_str) == Some(COMPACTION_PURPOSE) } fn hash_str(value: &str) -> u64 { @@ -429,16 +342,28 @@ fn segment_str<'a>(segment: &'a Value, key: &str) -> Option<&'a str> { #[cfg(test)] mod tests { + + /// The role of a message item, else the item kind, as the API reports it. + fn shown(item: &TrajectoryItem) -> &str { + item.role.as_deref().unwrap_or(match item.kind { + TrajectoryItemKind::Message => "message", + TrajectoryItemKind::Text => "text", + TrajectoryItemKind::ToolCall => "tool_call", + TrajectoryItemKind::ToolResult => "tool_result", + TrajectoryItemKind::Compaction => "compaction", + }) + } + #[test] fn usage_retains_events_without_new_content_and_counts_each_event_once() { - let body = r#"{"messages":[{"id":null,"role":"user","text":"hello"}],"instructions_present":false,"previous_response_id":null}"#; + let body = r#"{"messages":[{"id":null,"role":"user","text":"hello"}],"instructions_present":false}"#; let mut request = structured_event("request", body); request.input_tokens = 2; request.cache_read_tokens = 77_000; request.cache_write_tokens = 400; request.total_tokens = 77_402; let repeated = structured_event("request", body); - let mut response = event("response", "", json!({"token_usage_estimated": true})); + let mut response = event("response", "", json!({})); response.llm_model = "other-model".to_owned(); response.output_tokens = 150; response.reasoning_tokens = 40; @@ -463,32 +388,23 @@ mod tests { assert_eq!(trajectory.events[2].llm_model, "other-model"); assert_eq!(trajectory.events[2].output_tokens, 150); assert_eq!(trajectory.events[2].reasoning_tokens, 40); - assert!(trajectory.events[2].token_usage_estimated); + assert!(trajectory.events[2].token_usage_available); let serialized = serde_json::to_value(&trajectory.events).unwrap(); assert!(serialized[0].get("text").is_none()); assert!(serialized[0].get("metadata").is_none()); } #[test] - fn usage_distinguishes_reported_zero_from_absent_and_hidden_counts() { - let reported = event( - "response", - "", - json!({"token_usage_source": "provider_reported"}), - ); - let absent = event("request", "", json!({"token_usage_source": "absent"})); - let mut hidden = event( - "request", - "", - json!({"content_policy": {"token_usage": false}}), - ); - hidden.input_tokens = 10; - hidden.total_tokens = 10; - let trajectory = build(timeline(vec![reported, absent, hidden])); - assert_eq!(trajectory.events.len(), 3); - assert!(trajectory.events[0].token_usage_available); - assert!(!trajectory.events[1].token_usage_available); - assert!(!trajectory.events[2].token_usage_available); + fn usage_is_available_when_any_counter_is_non_zero() { + let zero = event("response", "", json!({})); + let mut cached = event("request", "", json!({})); + cached.cache_read_tokens = 10; + let trajectory = build(timeline(vec![zero, cached])); + assert_eq!(trajectory.events.len(), 2); + assert!(!trajectory.events[0].token_usage_available); + assert!(trajectory.events[1].token_usage_available); + let serialized = serde_json::to_value(&trajectory.events).unwrap(); + assert!(serialized[0].get("token_usage_estimated").is_none()); } use serde_json::json; @@ -527,17 +443,8 @@ mod tests { let items = build(timeline(events)).items; - let kinds = items.iter().map(|item| item.kind).collect::>(); - assert_eq!( - kinds, - [ - TrajectoryItemKind::Request, - TrajectoryItemKind::ToolCall, - TrajectoryItemKind::Request, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::Request, - ] - ); + let kinds = items.iter().map(shown).collect::>(); + assert_eq!(kinds, ["text", "tool_call", "text", "text", "text",]); assert_eq!(items[1].output.as_deref(), Some("README.md")); assert_eq!( items[4].text.as_deref(), @@ -566,11 +473,7 @@ mod tests { let items = build(timeline(events)).items; assert_eq!(items.len(), 3); - assert!( - items - .iter() - .all(|i| i.kind == TrajectoryItemKind::Request && !i.continued) - ); + assert!(items.iter().all(|i| shown(i) == "text" && !i.continued)); assert_eq!( items[2].text, Some(format!( @@ -591,27 +494,23 @@ mod tests { json!({"conversation":{"completeness":"partial","messages":[]}}), json!({"conversation":{"messages":"invalid"}}), json!({"conversation":{"messages":[],"completeness":"complete"}}), - // Older collectors stored messages without completeness. + // Without completeness the conversation is outside the contract. json!({"conversation":{"messages":[{"role":"user","text":"用户内容"}]}}), ] { let items = build(timeline(vec![event("request", text, metadata)])).items; assert_eq!(items.len(), 1); - assert_eq!(items[0].kind, TrajectoryItemKind::Request); + assert_eq!(shown(&items[0]), "text"); assert_eq!(items[0].text.as_deref(), Some(text)); assert!(!items[0].continued); } - let items = build(timeline(vec![structured_event("request", r#"{"messages":[{"id":null,"role":"unknown","text":"system: unknown"},{"id":null,"role":"unrecognized","text":"hello"},{"id":null,"role":"user","text":"system: quoted text"}],"instructions_present":false,"previous_response_id":null}"#)])).items; + let items = build(timeline(vec![structured_event("request", r#"{"messages":[{"id":null,"role":"unknown","text":"system: unknown"},{"id":null,"role":"unrecognized","text":"hello"},{"id":null,"role":"user","text":"system: quoted text"}],"instructions_present":false}"#)])).items; assert_eq!( - items.iter().map(|i| i.kind).collect::>(), - [ - TrajectoryItemKind::Request, - TrajectoryItemKind::Request, - TrajectoryItemKind::User - ] + items.iter().map(shown).collect::>(), + ["unknown", "unrecognized", "user"] ); } - /// A current collector always declares completeness; default it to complete. + /// Every conversation in the contract declares completeness; default it to complete. fn structured_event(side: &str, body: &str) -> UsageEventResponse { let mut conversation = serde_json::from_str::(body).unwrap(); conversation @@ -621,16 +520,16 @@ mod tests { .or_insert_with(|| json!("complete")); event( side, - "legacy fallback", + "flattened text", json!({"conversation": conversation}), ) } #[test] fn structured_projection_keeps_roles_and_repeated_user_occurrences() { - let first = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true,"previous_response_id":null}"#; - let response = r#"{"messages":[{"id":"a1","role":"assistant","text":"working"}],"instructions_present":false,"previous_response_id":null}"#; - let replay = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"},{"id":"a1","role":"assistant","text":"working"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true,"previous_response_id":null}"#; + let first = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true}"#; + let response = r#"{"messages":[{"id":"a1","role":"assistant","text":"working"}],"instructions_present":false}"#; + let replay = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"},{"id":"a1","role":"assistant","text":"working"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true}"#; let events = vec![ structured_event("request", first), structured_event("response", response), @@ -650,13 +549,8 @@ mod tests { } } assert_eq!( - merger.items.iter().map(|i| i.kind).collect::>(), - [ - TrajectoryItemKind::Context, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User - ] + merger.items.iter().map(shown).collect::>(), + ["developer", "user", "assistant", "user"] ); assert_eq!(merger.items[1].text.as_deref(), Some("继续")); assert_eq!(merger.items[3].text.as_deref(), Some("继续")); @@ -673,27 +567,22 @@ mod tests { fn streamed_anthropic_response_is_not_replayed_as_user_input() { let response = r#"{"messages":[{"role":"assistant","text":"hi"}]}"#; let items = build(timeline(vec![ - structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#), + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#), structured_event("response", response), - structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#), + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#), ])).items; assert_eq!( - items.iter().map(|i| i.kind).collect::>(), - [ - TrajectoryItemKind::Context, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User - ] + items.iter().map(shown).collect::>(), + ["system", "user", "assistant", "user"] ); assert_eq!(items[2].text.as_deref(), Some("hi")); } #[test] fn anthropic_reasoning_is_shown_once_with_its_assistant_message() { - let first = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null,"completeness":"complete"}"#; - let response = r#"{"messages":[{"id":"msg_1","role":"reasoning","text":"thinking it over"},{"id":"msg_1","role":"assistant","text":"hi"}],"instructions_present":false,"previous_response_id":null,"completeness":"complete"}"#; - let replay = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"reasoning","text":"thinking it over"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"previous_response_id":null,"completeness":"complete"}"#; + let first = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"completeness":"complete"}"#; + let response = r#"{"messages":[{"id":"msg_1","role":"reasoning","text":"thinking it over"},{"id":"msg_1","role":"assistant","text":"hi"}],"instructions_present":false,"completeness":"complete"}"#; + let replay = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"reasoning","text":"thinking it over"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"completeness":"complete"}"#; let items = build(timeline(vec![ structured_event("request", first), structured_event("response", response), @@ -703,14 +592,14 @@ mod tests { assert_eq!( items .iter() - .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) + .map(|i| (shown(i), i.text.as_deref().unwrap_or_default())) .collect::>(), [ - (TrajectoryItemKind::Context, "instructions"), - (TrajectoryItemKind::User, "hello"), - (TrajectoryItemKind::Assistant, "thinking it over"), - (TrajectoryItemKind::Assistant, "hi"), - (TrajectoryItemKind::User, "next"), + ("system", "instructions"), + ("user", "hello"), + ("reasoning", "thinking it over"), + ("assistant", "hi"), + ("user", "next"), ] ); assert!(items.iter().all(|i| i.stream == 0)); @@ -718,96 +607,85 @@ mod tests { #[test] fn incremental_requests_keep_identical_new_user_messages() { - let items = build(timeline(vec![ - structured_event( - "request", - r#"{"messages":[{"role":"developer","text":"instructions"},{"id":null,"role":"user","text":"continue"}],"instructions_present":true,"previous_response_id":null}"#, - ), - structured_event("response", r#"{"messages":[{"id":null,"role":"assistant","text":"working"}],"instructions_present":false,"previous_response_id":null}"#), - structured_event( - "request", - r#"{"messages":[{"id":null,"role":"user","text":"continue"}],"instructions_present":false,"previous_response_id":"r1"}"#, - ), - ])) - .items; + let mut first = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"id":null,"role":"user","text":"continue"}]}"#, + ); + first.response_id = Some("r1".to_owned()); + let mut response = structured_event( + "response", + r#"{"messages":[{"id":null,"role":"assistant","text":"working"}]}"#, + ); + response.response_id = Some("r1".to_owned()); + let mut next = structured_event( + "request", + r#"{"messages":[{"id":null,"role":"user","text":"continue"}]}"#, + ); + next.previous_response_id = Some("r1".to_owned()); + let items = build(timeline(vec![first, response, next])).items; assert_eq!( - items.iter().map(|i| i.kind).collect::>(), - [ - TrajectoryItemKind::Context, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User - ] + items.iter().map(shown).collect::>(), + ["developer", "user", "assistant", "user"] ); assert!(items.iter().all(|i| i.stream == 0)); } #[test] - fn structured_request_retains_system_and_assistant_roles() { - let body = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"previous_response_id":null}"#; - let items = build(timeline(vec![ - structured_event("request", body), - structured_event("request", body), - ])) - .items; + fn a_continuation_shows_new_developer_messages_but_not_its_instructions() { + let mut first = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"role":"user","text":"start"}],"instructions_present":true}"#, + ); + first.response_id = Some("r1".to_owned()); + let mut response = structured_event( + "response", + r#"{"messages":[{"role":"assistant","text":"working"}]}"#, + ); + response.response_id = Some("r1".to_owned()); + // The continuation repeats its instructions, then adds a developer + // notice and the next user input. + let mut next = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"role":"developer","text":"permissions changed"},{"role":"user","text":"go on"}],"instructions_present":true}"#, + ); + next.previous_response_id = Some("r1".to_owned()); + let items = build(timeline(vec![first, response, next])).items; assert_eq!( - items.iter().map(|i| i.kind).collect::>(), + texts(&items), [ - TrajectoryItemKind::Context, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User + ("developer", 0, "instructions"), + ("user", 0, "start"), + ("assistant", 0, "working"), + ("developer", 0, "permissions changed"), + ("user", 0, "go on"), ] ); } #[test] - fn changing_billing_headers_and_token_budgets_keep_one_stream() { - let request = |hash: &str, left: u32, messages: &str| { - let mut messages: Vec = serde_json::from_str(messages).unwrap(); - messages.insert(0, json!({"role":"system", "text": format!("x-anthropic-billing-header: cc_version=2.1.288; cch={hash}; cc_prev_req=req_{hash};\ninstructions\n\n{left} tokens left")})); - json!({"messages": messages, "instructions_present": true}).to_string() - }; + fn structured_request_retains_system_and_assistant_roles() { + let body = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#; let items = build(timeline(vec![ - structured_event( - "request", - &request("a1", 900, r#"[{"role":"user","text":"hello"}]"#), - ), - structured_event("response", r#"{"messages":[{"id":null,"role":"assistant","text":"hi"}],"instructions_present":false,"previous_response_id":null}"#), - structured_event( - "request", - &request( - "b2", - 850, - r#"[{"role":"user","text":"hello"},{"role":"assistant","text":"hi"},{"role":"user","text":"next"}]"#, - ), - ), + structured_event("request", body), + structured_event("request", body), ])) .items; - assert_eq!( - items - .iter() - .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) - .collect::>(), - [ - ( - TrajectoryItemKind::Context, - "x-anthropic-billing-header: cc_version=2.1.288; cch=a1; cc_prev_req=req_a1;\ninstructions\n\n900 tokens left" - ), - (TrajectoryItemKind::User, "hello"), - (TrajectoryItemKind::Assistant, "hi"), - (TrajectoryItemKind::User, "next"), - ] + items.iter().map(shown).collect::>(), + ["system", "user", "assistant", "user"] ); - assert!(items.iter().all(|i| i.stream == 0)); } - /// Claude Code thread requests as the collector projects them: a - /// `create` carries the whole system prompt and history, a `continue` - /// only the billing header and the messages added since - /// `previous_message_id`. - fn claude_request(thread: Value, system: &[&str], messages: Value) -> UsageEventResponse { + /// Claude Code requests as the collector projects them: a `create` + /// carries the whole system prompt and history, a `continue` only the + /// billing header and the messages added since the response it names. + fn claude_request( + stream: Option<&str>, + previous: Option<&str>, + response: Option<&str>, + system: &[&str], + messages: Value, + ) -> UsageEventResponse { let system = std::iter::once( "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; cch=3ceb5;", ) @@ -817,33 +695,37 @@ mod tests { let mut conversation = json!({ "messages": [{"id": null, "role": "system", "text": system}], "instructions_present": true, - "previous_response_id": null, }); conversation["messages"] .as_array_mut() .unwrap() .extend(serde_json::from_value::>(messages).unwrap()); let mut request = structured_event("request", &conversation.to_string()); - if !thread.is_null() { - request.metadata["claude_thread"] = thread; - } + request.stream_id = stream.map(str::to_owned); + request.previous_response_id = previous.map(str::to_owned); + request.response_id = response.map(str::to_owned); request } - fn create(messages: Value) -> UsageEventResponse { + /// A main-loop request with its whole history; `response` names the + /// response its exchange produced. + fn create(response: &str, messages: Value) -> UsageEventResponse { claude_request( - json!({"type": "create"}), + Some("main"), + None, + Some(response), &["You are Claude Code.", "Main loop instructions."], messages, ) } - fn continue_from(previous: &str, messages: Value) -> UsageEventResponse { - claude_request( - json!({"type": "continue", "previous_message_id": previous}), - &[], - messages, - ) + /// A main-loop request continuing after `previous`. + fn continue_from( + previous: &str, + response: Option<&str>, + messages: Value, + ) -> UsageEventResponse { + claude_request(Some("main"), Some(previous), response, &[], messages) } fn user(text: &str) -> Value { @@ -867,6 +749,7 @@ mod tests { ] } + /// A response found only by the response id its exchange declares. fn streamed(id: &str, text: &str) -> UsageEventResponse { let conversation = json!({ "messages": [ @@ -874,26 +757,28 @@ mod tests { {"id": id, "role": "assistant", "text": text}, {"id": null, "role": "tool", "text": ""}, ], - "instructions_present": false, - "previous_response_id": null, }); - structured_event("response", &conversation.to_string()) + let mut response = structured_event("response", &conversation.to_string()); + response.stream_id = None; + response.response_id = Some(id.to_owned()); + response } - /// A `thread_not_found` error carries no messages. + /// A `thread_not_found` error carries no messages and no response id. fn thread_not_found() -> UsageEventResponse { let mut response = structured_event( "response", - r#"{"messages":[],"instructions_present":false,"previous_response_id":null,"completeness":"unsupported"}"#, + r#"{"messages":[],"completeness":"unsupported"}"#, ); + response.stream_id = None; response.text = None; response } - fn texts(items: &[TrajectoryItem]) -> Vec<(TrajectoryItemKind, usize, &str)> { + fn texts(items: &[TrajectoryItem]) -> Vec<(&str, usize, &str)> { items .iter() - .map(|i| (i.kind, i.stream, i.text.as_deref().unwrap_or_default())) + .map(|i| (shown(i), i.stream, i.text.as_deref().unwrap_or_default())) .collect() } @@ -902,9 +787,11 @@ mod tests { instructions."; #[test] - fn thread_continue_appends_new_messages_in_the_stream_it_continues() { + fn a_continuation_appends_new_messages_in_the_stream_of_its_response() { let search = claude_request( - Value::Null, + None, + None, + Some("msg_side"), &[ "You are Claude Code.", "You are an assistant for performing a web search tool use", @@ -912,44 +799,58 @@ mod tests { json!([user("search the web")]), ); let events = vec![ - create(json!([user("hello"), budget()])), + create("msg_1", json!([user("hello"), budget()])), streamed("msg_1", "looking"), search, streamed("msg_side", "results"), - continue_from("msg_1", json!([tool_result(), budget()])), + continue_from("msg_1", Some("msg_2"), json!([tool_result(), budget()])), streamed("msg_2", "found it"), - continue_from("msg_2", json!([tool_result(), user("thanks"), budget()])), + continue_from( + "msg_2", + Some("msg_3"), + json!([tool_result(), user("thanks"), budget()]), + ), ]; let items = build(timeline(events)).items; assert_eq!( texts(&items), [ - (TrajectoryItemKind::Context, 0, MAIN_CONTEXT), - (TrajectoryItemKind::User, 0, "hello"), + ("system", 0, MAIN_CONTEXT), + ("user", 0, "hello"), ( - TrajectoryItemKind::Context, + "system", 0, "15000000 tokens left" ), - (TrajectoryItemKind::Assistant, 0, "looking"), + ("assistant", 0, "looking"), ( - TrajectoryItemKind::Context, + "system", 1, "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ cch=3ceb5;\nYou are Claude Code.\nYou are an assistant for performing a web \ search tool use" ), - (TrajectoryItemKind::User, 1, "search the web"), - (TrajectoryItemKind::Assistant, 1, "results"), - (TrajectoryItemKind::Assistant, 0, "found it"), - (TrajectoryItemKind::User, 0, "thanks"), + ("user", 1, "search the web"), + ("assistant", 1, "results"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "found it"), + ("user", 0, "thanks"), + ( + "system", + 0, + "15000000 tokens left" + ), ] ); } #[test] - fn thread_replay_after_a_failed_continue_adds_no_duplicates() { + fn a_replay_after_a_failed_continuation_adds_no_duplicates() { let first = user("hello"); let retry = user("try the wire"); let [looking_thinking, looking, looking_tool] = replayed("looking"); @@ -969,15 +870,15 @@ mod tests { budget(), ]); let events = vec![ - create(json!([first, budget()])), + create("msg_1", json!([first, budget()])), streamed("msg_1", "looking"), - continue_from("msg_1", json!([tool_result(), budget()])), + continue_from("msg_1", Some("msg_2"), json!([tool_result(), budget()])), streamed("msg_2", "found it"), - continue_from("msg_2", json!([retry, budget()])), + continue_from("msg_2", None, json!([retry, budget()])), thread_not_found(), - create(replay), + create("msg_3", replay), streamed("msg_3", "no wire port"), - continue_from("msg_3", json!([tool_result(), budget()])), + continue_from("msg_3", Some("msg_4"), json!([tool_result(), budget()])), streamed("msg_4", "compressed"), ]; let trajectory = build(timeline(events)); @@ -985,182 +886,306 @@ mod tests { assert_eq!( texts(&trajectory.items), [ - (TrajectoryItemKind::Context, 0, MAIN_CONTEXT), - (TrajectoryItemKind::User, 0, "hello"), + ("system", 0, MAIN_CONTEXT), + ("user", 0, "hello"), ( - TrajectoryItemKind::Context, + "system", 0, "15000000 tokens left" ), - (TrajectoryItemKind::Assistant, 0, "looking"), - (TrajectoryItemKind::Assistant, 0, "found it"), - (TrajectoryItemKind::User, 0, "try the wire"), - (TrajectoryItemKind::Assistant, 0, "no wire port"), - (TrajectoryItemKind::Assistant, 0, "compressed"), + ("assistant", 0, "looking"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "found it"), + ("user", 0, "try the wire"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "no wire port"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "compressed"), ] ); assert_eq!(trajectory.events.len(), 10); } #[test] - fn thread_continue_of_an_unknown_message_follows_the_latest_thread() { - let events = vec![ - create(json!([user("hello"), budget()])), + fn a_continuation_of_an_unseen_response_uses_its_stream_id_or_opens_a_stream() { + let title = || { claude_request( - Value::Null, + None, + None, + Some("msg_title"), &["You are a title generator."], json!([user("name it")]), - ), - continue_from("msg_uncaptured", json!([user("next"), budget()])), + ) + }; + let events = vec![ + create("msg_1", json!([user("hello"), budget()])), + title(), + continue_from("msg_uncaptured", None, json!([user("next"), budget()])), ]; let items = build(timeline(events)).items; assert_eq!( - items.last().map(|i| (i.kind, i.stream, i.text.as_deref())), - Some((TrajectoryItemKind::User, 0, Some("next"))) + texts(&items)[items.len() - 2..], + [ + ("user", 0, "next"), + ( + "system", + 0, + "15000000 tokens left" + ), + ] ); - // Without any earlier thread request the continue keeps its own stream. + // Without a stream id the continuation opens a stream of its own and + // shows only the messages it carries. + let mut orphan = continue_from("msg_uncaptured", None, json!([user("next")])); + orphan.stream_id = None; + let items = build(timeline(vec![title(), orphan])).items; + assert_eq!(texts(&items)[2..], [("user", 1, "next")]); + } + + #[test] + fn declared_stream_ids_keep_each_thread_apart() { + let request = |stream: &str, messages: &str| { + let mut event = structured_event( + "request", + &format!( + r#"{{"messages":[{{"role":"developer","text":"{stream} rules"}},{messages}]}}"# + ), + ); + event.stream_id = Some(stream.to_owned()); + event + }; let items = build(timeline(vec![ - claude_request( - Value::Null, - &["You are a title generator."], - json!([user("name it")]), + request("main", r#"{"role":"user","text":"fix it"}"#), + request("guardian", r#"{"role":"user","text":"review the action"}"#), + request( + "main", + r#"{"role":"user","text":"fix it"},{"role":"user","text":"and test"}"#, ), - continue_from("msg_uncaptured", json!([user("next")])), + request( + "guardian", + r#"{"role":"user","text":"review the action"},{"role":"user","text":"next action"}"#, + ), + ])) + .items; + + assert_eq!( + texts(&items), + [ + ("developer", 0, "main rules"), + ("user", 0, "fix it"), + ("developer", 1, "guardian rules"), + ("user", 1, "review the action"), + ("user", 0, "and test"), + ("user", 1, "next action"), + ] + ); + } + + #[test] + fn requests_without_a_stream_id_each_open_a_stream_and_repeat_their_history() { + let recap = || { + let mut event = structured_event( + "request", + r#"{"messages":[{"role":"system","text":"rules"},{"role":"user","text":"hello"},{"role":"user","text":"recap"}]}"#, + ); + event.stream_id = None; + event + }; + let items = build(timeline(vec![recap(), recap()])).items; + + assert_eq!( + texts(&items), + [ + ("system", 0, "rules"), + ("user", 0, "hello"), + ("user", 0, "recap"), + ("system", 1, "rules"), + ("user", 1, "hello"), + ("user", 1, "recap"), + ] + ); + } + + #[test] + fn a_response_joins_the_stream_of_the_request_with_its_response_id() { + let mut main = create("msg_main", json!([user("hello")])); + main.stream_id = Some("main".to_owned()); + let side = claude_request( + None, + None, + Some("msg_side"), + &["You are a summarizer."], + json!([user("summarize")]), + ); + // The main exchange answers after the side request was observed. + let items = build(timeline(vec![ + main, + side, + streamed("msg_main", "hi"), + streamed("msg_side", "summary"), + streamed("msg_unpaired", "orphan"), ])) .items; + assert_eq!( texts(&items)[2..], [ ( - TrajectoryItemKind::Context, + "system", 1, "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ - cch=3ceb5;" + cch=3ceb5;\nYou are a summarizer." ), - (TrajectoryItemKind::User, 1, "next"), + ("user", 1, "summarize"), + ("assistant", 0, "hi"), + ("assistant", 1, "summary"), + ("assistant", 2, "orphan"), ] ); } #[test] - fn system_messages_inside_the_conversation_keep_one_stream() { - let first = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"}],"instructions_present":true,"previous_response_id":null}"#; - let second = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"system","text":"reminder two"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"previous_response_id":null}"#; + fn opening_instructions_show_only_where_they_change() { + let request = |rules: &str, messages: &str| { + structured_event( + "request", + &format!( + r#"{{"messages":[{{"role":"system","text":"base"}},{{"role":"developer","text":"{rules}"}},{messages}],"instructions_present":true}}"# + ), + ) + }; let items = build(timeline(vec![ - structured_event("request", first), - structured_event("request", second), + request("rules v1", r#"{"role":"user","text":"hello"}"#), + request( + "rules v1", + r#"{"role":"user","text":"hello"},{"role":"user","text":"again"}"#, + ), + request( + "rules v2", + r#"{"role":"user","text":"hello"},{"role":"user","text":"again"},{"role":"user","text":"more"}"#, + ), ])) .items; assert_eq!( - items - .iter() - .map(|i| (i.kind, i.text.as_deref().unwrap_or_default())) - .collect::>(), + texts(&items), [ - (TrajectoryItemKind::Context, "instructions"), - (TrajectoryItemKind::User, "hello"), - (TrajectoryItemKind::Context, "reminder one"), - (TrajectoryItemKind::Assistant, "hi"), - (TrajectoryItemKind::Context, "reminder two"), - (TrajectoryItemKind::User, "next"), + ("system", 0, "base"), + ("developer", 0, "rules v1"), + ("user", 0, "hello"), + ("user", 0, "again"), + ("developer", 0, "rules v2"), + ("user", 0, "more"), ] ); - assert!(items.iter().all(|i| i.stream == 0)); } #[test] - fn structured_compaction_keeps_existing_summary_presentation() { - let request = json!({"messages": [{"role": "user", "text": "write report"}, {"role": "user", "text": PROMPT}]}).to_string(); - let replay = json!({"messages": [{"role": "user", "text": "write report"}, {"role": "user", "text": handoff()}]}).to_string(); + fn system_messages_inside_the_conversation_keep_one_stream() { + let first = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"}],"instructions_present":true}"#; + let second = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"system","text":"reminder two"},{"id":null,"role":"user","text":"next"}],"instructions_present":true}"#; let items = build(timeline(vec![ - structured_event("request", &request), - event("response", SUMMARY, json!({})), - structured_event("request", &replay), + structured_event("request", first), + structured_event("request", second), ])) .items; + assert_eq!( - items.iter().map(|i| i.kind).collect::>(), - [TrajectoryItemKind::User, TrajectoryItemKind::Compaction] + items + .iter() + .map(|i| (shown(i), i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + ("system", "instructions"), + ("user", "hello"), + ("system", "reminder one"), + ("assistant", "hi"), + ("system", "reminder two"), + ("user", "next"), + ] ); - assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); + assert!(items.iter().all(|i| i.stream == 0)); } const PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION. Create a \ handoff summary for another LLM that will resume the task."; const SUMMARY: &str = "## Goal\n\nFinish the report."; - fn handoff() -> String { - format!( - "Another language model started to solve this problem and produced a summary \ - of its thinking process. Here is the summary produced by the other language \ - model, use the information in this summary to assist with your own \ - analysis:\n{SUMMARY}" - ) + /// One exchange of a call the collector declares as a compaction. + fn compaction(texts: &[&str], response_id: &str) -> [UsageEventResponse; 2] { + let mut request = user_request(texts); + request.metadata["purpose"] = json!(COMPACTION_PURPOSE); + request.response_id = Some(response_id.to_owned()); + let mut response = event("response", SUMMARY, json!({"purpose": COMPACTION_PURPOSE})); + response.response_id = Some(response_id.to_owned()); + [request, response] } #[test] - fn compaction_prompt_and_summary_form_one_item_and_the_replay_is_hidden() { - let events = vec![ - user_request(&["write the report"]), - event("response", "working", json!({})), - user_request(&[ + fn a_compaction_exchange_forms_one_item_and_its_replay_is_ordinary_input() { + let [request, response] = compaction( + &[ "write the report", "date", PROMPT, - ]), - event("response", SUMMARY, json!({})), - user_request(&["cwd", &handoff()]), + ], + "resp_compact", + ); + let replay = format!("Here is the summary of the earlier conversation:\n{SUMMARY}"); + let events = vec![ + user_request(&["write the report"]), + event("response", "working", json!({})), + request, + response, + user_request(&["cwd", &replay]), event("response", "continuing", json!({})), ]; let items = build(timeline(events)).items; - let kinds = items.iter().map(|item| item.kind).collect::>(); assert_eq!( - kinds, - [ - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - TrajectoryItemKind::User, - TrajectoryItemKind::Compaction, - TrajectoryItemKind::User, - TrajectoryItemKind::Assistant, - ] + items.iter().map(shown).collect::>(), + ["user", "text", "compaction", "user", "user", "text",] ); - // New input sent along with the prompt stays user input. + // The new input of the compaction request is the item's input. assert_eq!( - items[2].text.as_deref(), - Some("date") - ); - assert_eq!(items[3].input.as_deref(), Some(PROMPT)); - assert_eq!(items[3].output.as_deref(), Some(SUMMARY)); - // The summary comes from the response, not from the prompt's request. - assert!(items[3].output_event_pk.is_some()); - assert_ne!(items[3].output_event_pk, Some(items[3].event_pk)); - assert_eq!( - items[4].text.as_deref(), - Some("cwd") + items[2].input.as_deref(), + Some(&*format!( + "date\n\n{PROMPT}" + )) ); + // The summary comes from the response of the same exchange. + assert_eq!(items[2].output.as_deref(), Some(SUMMARY)); + assert!(items[2].output_event_pk.is_some()); + assert_ne!(items[2].output_event_pk, Some(items[2].event_pk)); + // The replayed summary is shown like any other new input. + assert_eq!(items[4].text.as_deref(), Some(&*replay)); } #[test] - fn unobserved_compaction_is_recovered_from_its_handoff() { - let events = vec![ - user_request(&[&format!("resume\n\n{}", handoff())]), - event("response", "continuing", json!({})), - ]; - - let items = build(timeline(events)).items; - - assert_eq!(items.len(), 3); - assert_eq!(items[0].kind, TrajectoryItemKind::User); - assert_eq!(items[0].text.as_deref(), Some("resume")); - assert_eq!(items[1].kind, TrajectoryItemKind::Compaction); - assert_eq!(items[1].input, None); - assert_eq!(items[1].output.as_deref(), Some(SUMMARY)); - assert_eq!(items[2].kind, TrajectoryItemKind::Assistant); + fn compaction_prompt_text_alone_is_ordinary_input() { + let items = build(timeline(vec![ + user_request(&[PROMPT]), + event("response", SUMMARY, json!({})), + ])) + .items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["user", "text"] + ); } fn timeline(events: Vec) -> SessionTimelineResponse { @@ -1216,6 +1241,9 @@ mod tests { reasoning_tokens: 0, total_tokens: 0, observed_at: Utc::now(), + stream_id: Some("main".to_owned()), + response_id: None, + previous_response_id: None, metadata, attachments: Vec::new(), } diff --git a/crates/abyss-backend/src/usage/trajectory/conversation.rs b/crates/abyss-backend/src/usage/trajectory/conversation.rs index d15b6b2..a97815d 100644 --- a/crates/abyss-backend/src/usage/trajectory/conversation.rs +++ b/crates/abyss-backend/src/usage/trajectory/conversation.rs @@ -1,11 +1,15 @@ //! Session trajectory projection from collector-produced structured messages. +//! +//! Each exchange is placed in a stream by the identities the collector +//! declares, compared only for equality: a request continuing a known +//! response joins that response's stream, else a request naming a stream +//! joins it, and any other request opens a stream of its own. A response +//! joins the stream of the request that produced it. -use std::collections::{HashMap, HashSet}; - -use serde_json::Value; +use std::collections::HashMap; use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; -use crate::usage::conversation::{Conversation, Message, Role}; +use crate::usage::conversation::{Conversation, Message}; impl Message { fn matches(&self, other: &Self) -> bool { @@ -16,214 +20,139 @@ impl Message { _ => true, } } +} - const fn kind(&self) -> TrajectoryItemKind { - match self.role { - Role::System | Role::Developer => TrajectoryItemKind::Context, - Role::User => TrajectoryItemKind::User, - Role::Assistant | Role::Reasoning => TrajectoryItemKind::Assistant, - Role::Tool | Role::Unknown => TrajectoryItemKind::Request, - } - } +/// The stream an event belongs to. +#[derive(Clone, Copy)] +pub(super) struct Placement { + pub(super) stream: usize, + /// The request carries only the messages added since the response it + /// continues. + incremental: bool, } #[derive(Default)] pub(super) struct ConversationProjection { - streams: HashMap, - /// Stream of each response by its provider message id, so an Anthropic - /// thread `continue` naming that message joins the stream it continues. - messages: HashMap, - /// Stream of the latest Anthropic thread request. - thread: Option, + streams: Vec, + /// Stream of each declared stream id. + keys: HashMap, + /// Stream of each exchange by the response id it declares. + responses: HashMap, } #[derive(Default)] struct StreamHistory { + /// Instructions the latest full request opened with. + opening: Vec, + /// Messages after the opening, as the stream has seen them so far. history: Vec, - contexts: HashSet<(Role, String)>, } impl ConversationProjection { + /// Places an event by the identities its exchange declares. + pub(super) fn place(&mut self, event: &UsageEventResponse) -> Placement { + let request = event.event_type == "request"; + let continued = if request { + event + .previous_response_id + .as_ref() + .and_then(|id| self.responses.get(id).copied()) + } else { + // The request of the same exchange registered the response id. + event + .response_id + .as_ref() + .and_then(|id| self.responses.get(id).copied()) + }; + let stream = match (continued, &event.stream_id) { + (Some(stream), _) => stream, + (None, Some(key)) => { + if let Some(&stream) = self.keys.get(key) { + stream + } else { + let stream = self.open(); + self.keys.insert(key.clone(), stream); + stream + } + } + (None, None) => self.open(), + }; + if let Some(id) = &event.response_id { + self.responses.entry(id.clone()).or_insert(stream); + } + Placement { + stream, + incremental: request && event.previous_response_id.is_some(), + } + } + + fn open(&mut self) -> usize { + self.streams.push(StreamHistory::default()); + self.streams.len().saturating_sub(1) + } + pub(super) fn project( &mut self, event: &UsageEventResponse, - stream: &mut usize, - stream_ids: &mut HashMap, + placement: Placement, ) -> Option> { - // Older collectors stored messages without completeness; their events keep - // the flattened-text presentation they had before structured conversations. - if !Conversation::declared(&event.metadata) { - return None; - } let conversation = Conversation::read(&event.metadata)?; let request = event.event_type == "request"; - let (messages, incremental) = if request { - let messages = conversation - .messages - .into_iter() - .filter(|m| m.role != Role::Tool) - .collect::>(); - if messages.is_empty() { - return None; - } - let incremental = self.request_stream( - event, - conversation.previous_response_id.is_some(), - conversation.instructions_present, - &messages, - stream, - stream_ids, - ); - (messages, incremental) - } else { - if let Some(id) = conversation.messages.iter().find_map(|m| m.id.as_deref()) { - self.messages.insert(id.to_owned(), *stream); - } - let mut messages = conversation - .messages - .into_iter() - .filter(|m| m.role != Role::Tool) - .collect::>(); - if messages.is_empty() - && let Some(text) = event.text.as_ref().filter(|v| !v.is_empty()) - { - messages.push(Message { - id: None, - role: Role::Assistant, - text: text.clone(), - }); - } - (messages, false) - }; - // Legacy response text remains available when no message was projected. - if !request && messages.is_empty() { + // A request that declares its own instructions carries them as its + // first message. Messages without text, such as tool items whose + // content the tool fields carry, take no part in the trajectory. + let mut messages = conversation.messages; + let opening_len = usize::from(request && conversation.instructions_present); + let rest = messages + .split_off(opening_len.min(messages.len())) + .into_iter() + .filter(|message| !message.text.is_empty()) + .collect::>(); + let (opening, messages) = (messages, rest); + if opening.is_empty() && messages.is_empty() { return None; } - let state = self.streams.entry(*stream).or_default(); - let mut cursor = 0; - let mut next = Vec::new(); + let stream = placement.stream; + let state = self.streams.get_mut(stream)?; + let item = |message: &Message| { + let mut item = TrajectoryItem::new(TrajectoryItemKind::Message, stream, event); + item.role = Some(message.role.clone()); + item.text = Some(message.text.clone()); + item + }; let mut added = Vec::new(); - for message in messages { - let context = message.kind() == TrajectoryItemKind::Context; - let known = if context { - !state - .contexts - .insert((message.role, stable_context(&message.text))) - } else if request && !incremental { - let found = state.history[cursor..] - .iter() - .position(|old| old.matches(&message)); - if let Some(offset) = found { - cursor = cursor.saturating_add(offset).saturating_add(1); - } - found.is_some() - } else if request { - state.history.iter().any(|old| { - old.matches(&message) && (message.id.is_some() || message.role != Role::User) - }) - } else { - false - }; - if !context { - if request && !incremental { - next.push(message.clone()); - } else if !known { - state.history.push(message.clone()); - } - } - if !known && !message.text.is_empty() { - let mut item = TrajectoryItem::new(message.kind(), *stream, event); - item.text = Some(message.text); - added.push(item); + // A response and a continuation carry only messages added since the + // response they continue, so each one is new; the history keeps them + // for a later full request that replays them. A continuation does not + // repeat the conversation's instructions. + if !request || placement.incremental { + for message in messages { + added.push(item(&message)); + state.history.push(message); } + return Some(added); } - if request && !incremental { - state.history = next; + + // A full request shows its instructions only where they changed, and + // of its history only what the stream has not seen in that order. + for message in &opening { + if !message.text.is_empty() && !state.opening.iter().any(|old| old.matches(message)) { + added.push(item(message)); + } } - Some(added) - } + state.opening = opening; - /// Points `stream` at the stream this request belongs to and tells - /// whether the request sends only messages added since an earlier one. - fn request_stream( - &mut self, - event: &UsageEventResponse, - responds_to_previous: bool, - instructions_present: bool, - messages: &[Message], - stream: &mut usize, - stream_ids: &mut HashMap, - ) -> bool { - // An Anthropic thread `continue` sends only the messages added - // since the response it names, and a system prompt that is only - // the billing header, so it neither repeats nor names its stream. - let thread = event.metadata.get("claude_thread"); - let continued = thread - .and_then(|thread| thread.get("type")) - .and_then(Value::as_str) - == Some("continue"); - let incremental = continued || responds_to_previous; - let continued_stream = continued - .then(|| { - thread - .and_then(|thread| thread.get("previous_message_id")) - .and_then(Value::as_str) - .and_then(|id| self.messages.get(id).copied()) - // The named response was not captured: thread - // requests of a session follow one another. - .or(self.thread) - }) - .flatten(); - if let Some(continued) = continued_stream { - *stream = continued; - } else if !incremental || instructions_present { - // A stream is named by the prompt the conversation opens with. - // System messages later in the conversation are part of it: - // counting them made each new one start another stream. - let context = messages + let mut cursor = 0; + for message in &messages { + let found = state.history[cursor..] .iter() - .take_while(|message| message.kind() == TrajectoryItemKind::Context) - .map(|message| { - format!( - "{}: {}", - message.role.as_str(), - stable_context(&message.text) - ) - }) - .collect::>() - .join("\n\n"); - let next = stream_ids.len(); - *stream = *stream_ids.entry(super::hash_str(&context)).or_insert(next); - } - if thread.is_some() { - self.thread = Some(*stream); + .position(|old| old.matches(message)); + match found { + Some(offset) => cursor = cursor.saturating_add(offset).saturating_add(1), + None => added.push(item(message)), + } } - incremental + state.history = messages; + Some(added) } } - -/// Claude Code adds two lines to its system prompt that change on every -/// request: a leading billing header whose request hash and previous-request -/// id differ each time, for example `x-anthropic-billing-header: -/// cc_version=2.1.288.9e5; cc_entrypoint=cli; cch=e4f32; cc_is_subagent=true; -/// cc_prev_req=req_…; cc_prompt_id=…;`, and a trailing context budget such as -/// `14964936 tokens left`. Counting them made -/// every request a new stream that replayed the whole conversation, so -/// streams and system prompts are told apart without them. The stored conversation -/// keeps both, so they can still be counted for billing. -const BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:"; -const TOKEN_BUDGET_OPEN: &str = ""; -const TOKEN_BUDGET_CLOSE: &str = ""; - -/// A system prompt without the lines that change on every request. -fn stable_context(text: &str) -> String { - text.lines() - .filter(|line| !changes_every_request(line.trim())) - .collect::>() - .join("\n") -} - -fn changes_every_request(line: &str) -> bool { - line.starts_with(BILLING_HEADER_PREFIX) - || (line.starts_with(TOKEN_BUDGET_OPEN) && line.ends_with(TOKEN_BUDGET_CLOSE)) -} From 366ac5e6a0cabd5c519c499bde1c364fc92575b5 Mon Sep 17 00:00:00 2001 From: Auceptin Date: Wed, 14 Oct 2026 05:37:57 -0700 Subject: [PATCH 30/30] test(blackbox): follow the AgentEvent ingest contract - Fixtures send `codex` / `openai` as abyss-rs does, explicit token counters through a `token_usage()` helper (totals are the disjoint sum, so the summary total is 36), and exchange identities at the top level instead of in metadata. - New checks: `agent_name=Codex`, `agent_name=claude-desktop`, and `llm_provider=OpenAI` match nothing, and events return the exchange identities they were ingested with. - `claude_turns_api.py` becomes `turns_api.py`, covering `turn_id` placement, adoption of the opening turn, and numbering by start time. - Title fixtures mark naming threads with `naming_thread`; the mock Elasticsearch drops the commands and file paths fields. Co-Authored-By: Claude Opus 5.5 --- scripts/tests/blackbox_api.py | 82 ++++++++++++++++++++++------- scripts/tests/claude_turns_api.py | 76 -------------------------- scripts/tests/mock_elasticsearch.py | 2 - scripts/tests/session_titles_api.py | 22 ++++---- scripts/tests/turns_api.py | 58 ++++++++++++++++++++ 5 files changed, 133 insertions(+), 107 deletions(-) delete mode 100644 scripts/tests/claude_turns_api.py create mode 100644 scripts/tests/turns_api.py diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index 00c6074..e2b6c5d 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -69,6 +69,24 @@ def require_status(response: HttpResponse, expected: int, context: str) -> None: ) +def token_usage( + input_tokens: int = 0, + output_tokens: int = 0, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, +) -> dict[str, int]: + """AgentEvent token counters; the total is the sum of the disjoint four.""" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + "reasoning_tokens": reasoning_tokens, + "total_tokens": input_tokens + output_tokens + cache_read_tokens + cache_write_tokens, + } + + def event( event_id: str, session_id: str, @@ -85,19 +103,15 @@ def event( "platform": "linux", "os_version": "blackbox", }, - "agent": {"name": "Codex", "version": "blackbox"}, + "agent": {"name": "codex", "version": "blackbox"}, "session_id": session_id, - "turn_index": 1, - "llm": {"provider": "OpenAI", "model": "gpt-5.5"}, + "stream_id": f"stream-{run_id}", + "response_id": f"response-{run_id}", + "llm": {"provider": "openai", "model": "gpt-5.5"}, "event_type": event_type, "text": text, "token_usage": token_usage, - "metadata": { - "test_run": run_id, - "response_id": f"response-{run_id}", - "previous_response_id": None, - "provider_call_index": 1, - }, + "metadata": {"test_run": run_id}, } if event_type == "request": value["attachments"] = [ @@ -136,7 +150,7 @@ def run(base_url: str, token: str, run_id: str) -> None: session_id, "request", "blackbox request", - {"input_tokens": 12, "cache_read_tokens": 3}, + token_usage(input_tokens=12, cache_read_tokens=3), run_id, ), event( @@ -144,7 +158,7 @@ def run(base_url: str, token: str, run_id: str) -> None: session_id, "response", "blackbox response", - {"output_tokens": 21, "reasoning_tokens": 4, "total_tokens": 25}, + token_usage(output_tokens=21, reasoning_tokens=4), run_id, ), ], @@ -215,6 +229,15 @@ def run(base_url: str, token: str, run_id: str) -> None: IMAGE_BASE64.encode() not in raw_response.body, "raw event responses must not expose attachment content", ) + require( + all( + item.get("stream_id") == f"stream-{run_id}" + and item.get("response_id") == f"response-{run_id}" + and item.get("previous_response_id") is None + for item in raw_events + ), + "events must return the exchange identities they were ingested with", + ) # The list projection carries what an event list shows and no content. list_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( @@ -273,10 +296,10 @@ def run(base_url: str, token: str, run_id: str) -> None: require(len(rows) == 1, "usage summary must contain one aggregate row") summary = rows[0] require(summary["user_id"] == OWNER_ID, "summary must belong to the owner") - require(summary["agent_name"] == "codex", "summary must normalize Agent names") - require(summary["llm_provider"] == "openai", "summary must normalize providers") + require(summary["agent_name"] == "codex", "summary must report the Agent name as received") + require(summary["llm_provider"] == "openai", "summary must report the provider as received") require(summary["requests"] == 1 and summary["responses"] == 1, "summary event counts are incorrect") - require(summary["total_tokens"] == 40, "summary token total is incorrect") + require(summary["total_tokens"] == 36, "summary token total is incorrect") # Repeating a filter key matches any of its values; filters still combine # with each other. @@ -286,9 +309,9 @@ def run(base_url: str, token: str, run_id: str) -> None: ("session_id", other_session_id), ("session_id", session_id), ("agent_name", "claude-code"), - ("agent_name", "Codex"), + ("agent_name", "codex"), ("llm_provider", "anthropic"), - ("llm_provider", "OpenAI"), + ("llm_provider", "openai"), ("llm_model", "gpt-5.5"), ("llm_model", "absent-model"), ("limit", 10), @@ -300,6 +323,27 @@ def run(base_url: str, token: str, run_id: str) -> None: len(any_of_response.json()["events"]) == 2, "repeated filter values must match any of the values", ) + # Agent names match exactly as reported: no case folding or aliases. + for agent in ["Codex", "claude-desktop"]: + exact_agent_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("agent_name", agent)] + ) + exact_agent_response = request(base_url, "GET", exact_agent_path, token=token) + require_status(exact_agent_response, 200, "raw event query with another Agent") + require( + exact_agent_response.json()["events"] == [], + f"agent_name={agent} must not match codex events", + ) + # Providers also match exactly as reported. + folded_provider_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("llm_provider", "OpenAI")] + ) + folded_provider_response = request(base_url, "GET", folded_provider_path, token=token) + require_status(folded_provider_response, 200, "raw event query with a case-folded provider") + require( + folded_provider_response.json()["events"] == [], + "llm_provider=OpenAI must not match the reported openai", + ) none_of_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( [("session_id", session_id), ("llm_model", "absent-model"), ("llm_model", "other-model")] ) @@ -322,7 +366,7 @@ def run(base_url: str, token: str, run_id: str) -> None: require_status(any_of_summary, 200, "usage summary with repeated filters") any_of_rows = any_of_summary.json()["rows"] require( - [(row["llm_model"], row["total_tokens"]) for row in any_of_rows] == [("gpt-5.5", 40)], + [(row["llm_model"], row["total_tokens"]) for row in any_of_rows] == [("gpt-5.5", 36)], "summary must aggregate every repeated filter value", ) any_of_sessions_path = "/v1/agent-usage/sessions?" + urllib.parse.urlencode( @@ -492,9 +536,9 @@ def run(base_url: str, token: str, run_id: str) -> None: run_title_contract(base_url, token, run_id) - from claude_turns_api import run_claude_turn_contract + from turns_api import run_turn_contract - run_claude_turn_contract(base_url, token, run_id) + run_turn_contract(base_url, token, run_id) print(f"blackbox API contract passed for {session_id}") diff --git a/scripts/tests/claude_turns_api.py b/scripts/tests/claude_turns_api.py deleted file mode 100644 index b5ef7cd..0000000 --- a/scripts/tests/claude_turns_api.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Claude Code turn attachment contract shared by both storage black-box suites.""" - -import urllib.parse - -from blackbox_api import event, request, require, require_status - - -def run_claude_turn_contract(base_url: str, token: str, run_id: str) -> None: - session_id = f"{run_id}-claude-thread" - - def make(name, second, metadata, *, agent="claude-code", session=session_id): - value = event(f"{run_id}-{name}", session, "request", name, {"input_tokens": 1}, run_id) - value.pop("attachments", None) - value["agent"]["name"] = agent - value["llm"] = {"provider": "Anthropic", "model": "claude-x"} - value["observed_at"] = f"2026-10-08T00:00:{second:02}Z" - # Current collectors send no turn number; blackbox_api's events still - # carry one, as older collectors do. - value.pop("turn_index") - value["metadata"] = metadata - return value - - def main_loop(turn, prompt): - return { - "claude_turn_id": f"cc_turn:{turn}", - "claude_code_billing": {"cc_prompt_id": prompt, "cc_turn_index": str(turn)}, - } - - def ingest(*events): - result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) - require_status(result, 200, "Claude turn fixture ingest") - - def turns(session): - query = urllib.parse.urlencode({"session_id": session, "fields": "list", "limit": 100}) - result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) - require_status(result, 200, "Claude turn event lookup") - events = result.json()["events"] - return {item["event_id"].removeprefix(f"{run_id}-"): item["turn_pk"] for item in events} - - ingest( - make("main-1", 10, main_loop(1, "prompt-1")), - make("subagent-1", 12, {"claude_code_billing": {"cc_prompt_id": "prompt-1", "cc_is_subagent": "true"}}), - make("recap", 20, {"message_id": f"msg-recap-{run_id}"}), - make("main-2", 30, main_loop(2, "prompt-2")), - make("main-2-replay", 35, main_loop(2, "prompt-2b")), - make("subagent-2", 40, {"claude_code_billing": {"cc_prompt_id": "prompt-2b"}}), - ) - ingest( - make("late-title", 32, {"request_hash": f"title-{run_id}"}), - make("late-subagent", 45, {"claude_code_billing": {"cc_prompt_id": "prompt-1"}}), - make("early-probe", 1, {}), - ) - turn = turns(session_id) - require(turn["main-1"] != turn["main-2"], "Claude Code main-loop turns stay distinct") - require(turn["main-2-replay"] == turn["main-2"], "a replay keeps its cc_turn turn") - for name in ["subagent-1", "recap", "late-subagent", "early-probe"]: - require(turn[name] == turn["main-1"], f"{name} joins the first turn") - for name in ["subagent-2", "late-title"]: - require(turn[name] == turn["main-2"], f"{name} joins the second turn") - - other = f"{run_id}-codex-turns" - ingest( - make("codex-1", 10, {"codex_turn_id": f"turn-{run_id}"}, agent="codex", session=other), - make("codex-2", 20, {"response_id": f"response-b-{run_id}"}, agent="codex", session=other), - make("codex-2-response", 21, {"turn_id": f"response-b-{run_id}"}, agent="codex", session=other), - make("unidentified-1", 30, {}, agent="codex", session=other), - make("unidentified-2", 31, {}, agent="codex", session=other), - ) - codex = turns(other) - require(codex["codex-1"] != codex["codex-2"], "other Agents keep provider turn identities") - require(codex["codex-2-response"] != codex["codex-1"], "a turn_id names its own turn") - require( - len({codex["codex-1"], codex["codex-2"], codex["unidentified-1"], codex["unidentified-2"]}) == 4, - "an event without any turn identity is a turn of its own", - ) - print("Claude Code turn attachment API contract passed") diff --git a/scripts/tests/mock_elasticsearch.py b/scripts/tests/mock_elasticsearch.py index a191690..1e907a5 100755 --- a/scripts/tests/mock_elasticsearch.py +++ b/scripts/tests/mock_elasticsearch.py @@ -17,8 +17,6 @@ HIGHLIGHT_END = "[[[ABYSS_SEARCH_HIGHLIGHT_END]]]" SEARCHABLE_FIELDS = ( "content", - "commands", - "file_paths", "tool_names", "tool_content", "session_id", diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py index 673a8ec..0cc9bba 100644 --- a/scripts/tests/session_titles_api.py +++ b/scripts/tests/session_titles_api.py @@ -4,7 +4,7 @@ import urllib.parse import uuid -from blackbox_api import event, request, require, require_status +from blackbox_api import event, request, require, require_status, token_usage PROMPT = "Draw a pelican riding a bicycle" @@ -12,17 +12,18 @@ def run_title_contract(base_url: str, token: str, run_id: str) -> None: - def make(session, kind, text, *, source="user", agent="codex", metadata=None): - value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, {}, run_id) + def make(session, kind, text, *, naming_thread=False, agent="codex", metadata=None): + value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, token_usage(), run_id) value.pop("attachments", None) value["agent"]["name"] = agent value["device"]["host_name"] = f"title-contract-{run_id}" value["observed_at"] = "2026-09-30T00:00:01Z" if kind == "request" else "2026-09-30T00:00:02Z" value["metadata"] = dict(metadata or {}) + if naming_thread: + value["naming_thread"] = True if kind == "request": value["metadata"]["conversation"] = { - "messages": [{"role": "user", "text": text}], - "thread_source": source, "completeness": "complete", + "messages": [{"role": "user", "text": text}], "completeness": "complete", } return value @@ -57,12 +58,13 @@ def session(name): require(value["title"] == "Build fix", f"{name}: the collector title names the session") require(value["purpose"] is None, f"{name}: a session naming itself is not a title thread") - # Codex: the naming thread is marked; its task keeps its first user input. - ingest(make("naming", "request", CODEX_NAMING, source="thread_title"), - make("naming", "response", '{"title":"Draw a pelican"}', source="thread_title")) + # A thread opened only to name another session is marked; the task keeps + # its first user input. + ingest(make("naming", "request", CODEX_NAMING, naming_thread=True), + make("naming", "response", '{"title":"Draw a pelican"}', naming_thread=True)) value = session("naming") - require(value["purpose"] == "title_generation", "Codex naming threads are marked") - require(value["title"] is None and value["parent_session_pk"] is None, "Codex naming threads are not linked") + require(value["purpose"] == "title_generation", "naming threads are marked") + require(value["title"] is None and value["parent_session_pk"] is None, "naming threads are not linked") ingest(make("task", "request", PROMPT)) require(session("task")["title"] == PROMPT, "Codex tasks keep their first user input") diff --git a/scripts/tests/turns_api.py b/scripts/tests/turns_api.py new file mode 100644 index 0000000..f43ee26 --- /dev/null +++ b/scripts/tests/turns_api.py @@ -0,0 +1,58 @@ +"""Turn placement contract shared by both storage black-box suites.""" + +import urllib.parse + +from blackbox_api import event, request, require, require_status, token_usage + + +def run_turn_contract(base_url: str, token: str, run_id: str) -> None: + def make(session, name, second, turn_id, *, agent="claude-code"): + value = event(f"{run_id}-{name}", session, "request", name, token_usage(input_tokens=1), run_id) + value.pop("attachments", None) + value["agent"]["name"] = agent + value["llm"] = {"provider": "Anthropic", "model": "claude-x"} + value["observed_at"] = f"2026-10-08T00:00:{second:02}Z" + value["metadata"] = {} + if turn_id is not None: + value["turn_id"] = turn_id + return value + + def ingest(*events): + result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) + require_status(result, 200, "turn fixture ingest") + + def turns(session): + query = urllib.parse.urlencode({"session_id": session, "fields": "list", "limit": 100}) + result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) + require_status(result, 200, "turn event lookup") + events = result.json()["events"] + return {item["event_id"].removeprefix(f"{run_id}-"): item["turn_index"] for item in events} + + # Events naming a turn share it; events naming none join the turn in + # progress; the first named turn adopts the turn opened without one. + session = f"{run_id}-turns" + ingest( + make(session, "probe", 5, None), + make(session, "main-1", 10, "p1"), + make(session, "subagent-1", 12, "p1"), + make(session, "recap", 20, None), + make(session, "main-2", 30, "p2"), + ) + ingest( + make(session, "late-title", 32, None), + make(session, "late-subagent", 45, "p1"), + ) + turn = turns(session) + for name in ["probe", "main-1", "subagent-1", "recap", "late-subagent"]: + require(turn[name] == 1, f"{name} belongs to the first turn") + for name in ["main-2", "late-title"]: + require(turn[name] == 2, f"{name} belongs to the second turn") + + # Turns are numbered by start time, whatever order they arrive in, and + # without regard to the Agent. + late = f"{run_id}-late-turns" + ingest(make(late, "b", 30, "b", agent="codex"), make(late, "c", 50, "c", agent="codex")) + ingest(make(late, "a", 10, "a", agent="codex")) + turn = turns(late) + require((turn["a"], turn["b"], turn["c"]) == (1, 2, 3), "a late turn takes its place by start time") + print("turn placement API contract passed")