diff --git a/.gitignore b/.gitignore index c901f2f..9847753 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target/ +__pycache__/ .DS_Store .env diff --git a/Cargo.lock b/Cargo.lock index cad5933..eba7807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,6 +7,7 @@ name = "abyss-backend" version = "1.0.0" dependencies = [ "axum", + "axum-extra", "base64 0.21.7", "chrono", "diesel", @@ -110,6 +111,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-extra" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be44683b41ccb9ab2d23a5230015c9c3c55be97a25e4428366de8873103f7970" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "serde_core", + "serde_html_form", + "serde_path_to_error", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.21.7" @@ -279,9 +303,9 @@ checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "diesel" -version = "2.3.12" +version = "2.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715377c6e464cb44bb89bd8487584240516c8d5052bc645d6babc50bb8be46c3" +checksum = "ebe9f7eaef33febd60290c5a9f3b0571d03c4c04a24739cd64808ce30e65a5d3" dependencies = [ "bitflags", "byteorder", @@ -300,9 +324,9 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.3.9" +version = "2.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" +checksum = "ecbd51fb6c020672543641167efa4e6417ff7ad76849ed556ace3595e72de03a" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", @@ -378,6 +402,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -494,6 +524,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -745,6 +781,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -1185,7 +1231,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", "thiserror", ] @@ -1287,6 +1333,19 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "serde_html_form" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f2d7ff8a2140333718bb329f5c40fc5f0865b84c426183ce14c97d2ab8154f" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "ryu", + "serde_core", +] + [[package]] name = "serde_json" version = "1.0.151" diff --git a/README.md b/README.md index 0e37e07..0b6d535 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,16 @@ and optional full-text search. - Health and storage-readiness probes. - Embedded migrations for both supported storage profiles. +A session is titled by the `session_title` the collector reports for a call in +which the Agent names its own session (Claude Code); otherwise by its first +nonempty user input, collapsed to one line and truncated to 80 characters with +an ellipsis. An Agent title is never replaced by a default title. Requests whose +`purpose` marks them as naming calls are not user input. Codex names tasks from +separate threads that do not say which session they name; those threads are +marked `title_generation` from their `thread_source` and are not linked to a +task. Naming calls retain their own events and usage. Existing sessions receive +default titles through a one-time backfill. + Exactly one storage profile is compiled into a binary: - `postgres-es` uses PostgreSQL as the source of truth and optionally projects diff --git a/crates/abyss-backend/Cargo.toml b/crates/abyss-backend/Cargo.toml index e926122..89581e7 100644 --- a/crates/abyss-backend/Cargo.toml +++ b/crates/abyss-backend/Cargo.toml @@ -31,6 +31,7 @@ sqlite-fts = [ [dependencies] axum = "0.8" +axum-extra = { version = "0.12", default-features = false, features = ["query"] } base64 = "0.21" chrono = { version = "0.4", features = ["serde"] } diesel = { version = "2.2", default-features = false, features = ["32-column-tables", "r2d2", "with-deprecated"], optional = true } diff --git a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql similarity index 91% rename from crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql rename to crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql index 6c08204..fbf1f4a 100644 --- a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/down.sql +++ b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/down.sql @@ -1,3 +1,4 @@ +DROP TABLE session_titles; DROP TABLE usage_events_fts; DROP TABLE agent_diagnostic_capture_events; DROP TABLE agent_diagnostic_captures; diff --git a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql similarity index 73% rename from crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql rename to crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql index cafdfbf..483d88e 100644 --- a/crates/abyss-backend/migrations-sqlite/20260827000000_initial_schema/up.sql +++ b/crates/abyss-backend/migrations-sqlite/20261014150000_initial_schema/up.sql @@ -47,16 +47,21 @@ CREATE TABLE agent_sessions ( CREATE INDEX agent_sessions_user_time_idx ON agent_sessions (user_id, started_at DESC); +-- turn_key is the turn the Agent named for its requests, unique within the +-- session; it is NULL for the turn opened by requests naming none. turn_index +-- is the turn's position in the session by start time. CREATE TABLE agent_turns ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, session_pk TEXT NOT NULL REFERENCES agent_sessions(id) ON DELETE CASCADE, turn_index INTEGER NOT NULL CHECK (turn_index >= 1), + turn_key TEXT, started_at INTEGER NOT NULL, ended_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, - UNIQUE (session_pk, turn_index) + UNIQUE (session_pk, turn_index), + UNIQUE (session_pk, turn_key) ) STRICT; CREATE INDEX agent_turns_user_session_idx @@ -85,6 +90,12 @@ CREATE TABLE llm_usage_events ( reasoning_tokens INTEGER NOT NULL DEFAULT 0 CHECK (reasoning_tokens >= 0), total_tokens INTEGER NOT NULL DEFAULT 0 CHECK (total_tokens >= 0), observed_at INTEGER NOT NULL, + -- Agent-declared thread of the exchange; equal values continue one conversation. + stream_id TEXT, + -- Provider identity of the response the exchange produced. + response_id TEXT, + -- Response whose conversation the exchange's request continues. + previous_response_id TEXT, metadata TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata)), created_at INTEGER NOT NULL ) STRICT; @@ -154,7 +165,35 @@ CREATE VIRTUAL TABLE usage_events_fts USING fts5( content, tool_names, tool_content, - commands, - file_paths, tokenize = 'unicode61' ); + + +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads, +-- and parent_session_pk the task session such a thread was matched to. +CREATE TABLE session_titles ( + session_pk TEXT PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title TEXT, + title_event_pk TEXT REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at INTEGER, + purpose TEXT, + parent_session_pk TEXT REFERENCES agent_sessions(id) ON DELETE SET NULL, + updated_at INTEGER NOT NULL +) STRICT; + +-- Event rows hold whole request bodies in text and metadata, so reading a +-- token column from the row walks through megabytes of overflow pages. These +-- indexes carry every column the usage summary and the session list read, +-- letting both answer from the index alone. +CREATE INDEX llm_usage_events_summary_cover_idx + ON llm_usage_events ( + user_id, observed_at, agent_name, llm_provider, llm_model, event_type, + device_context_id, session_pk, turn_pk, session_id, + input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, + reasoning_tokens, total_tokens + ); +CREATE INDEX llm_usage_events_session_activity_idx + ON llm_usage_events (session_pk, observed_at, total_tokens); diff --git a/crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql b/crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql similarity index 92% rename from crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql rename to crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql index 1e929fa..06486fa 100644 --- a/crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql +++ b/crates/abyss-backend/migrations/20261014150000_initial_schema/down.sql @@ -1,5 +1,6 @@ -- Drop in reverse dependency order so every foreign key remains valid while -- the consolidated standalone schema is removed. +DROP TABLE IF EXISTS session_titles; DROP TABLE IF EXISTS search_outbox; DROP TABLE IF EXISTS agent_diagnostic_capture_events; DROP TABLE IF EXISTS agent_diagnostic_captures; diff --git a/crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql b/crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql similarity index 84% rename from crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql rename to crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql index 551ee1b..8faca4a 100644 --- a/crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql +++ b/crates/abyss-backend/migrations/20261014150000_initial_schema/up.sql @@ -33,7 +33,7 @@ CREATE TABLE devices ( CREATE INDEX devices_user_seen_idx ON devices (user_id, last_seen_at DESC); --- Sessions use the Agent-native session ID plus canonical Agent name. The +-- Sessions use the Agent-native session ID plus the reported Agent name. The -- device reference follows the most recently observed device for that session. CREATE TABLE agent_sessions ( id uuid PRIMARY KEY, @@ -53,18 +53,21 @@ CREATE TABLE agent_sessions ( CREATE INDEX agent_sessions_user_time_idx ON agent_sessions (user_id, started_at DESC); --- Turn indexes are backend-normalized during ingest when stable provider or --- Agent metadata reveals that a collector restarted its local counter. +-- turn_key is the turn the Agent named for its requests, unique within the +-- session; it is NULL for the turn opened by requests naming none. turn_index +-- is the turn's position in the session by start time. CREATE TABLE agent_turns ( id uuid PRIMARY KEY, user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, session_pk uuid NOT NULL REFERENCES agent_sessions(id) ON DELETE CASCADE, turn_index integer NOT NULL CHECK (turn_index >= 1), + turn_key text, started_at timestamptz NOT NULL, ended_at timestamptz, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), - UNIQUE (session_pk, turn_index) + UNIQUE (session_pk, turn_index), + UNIQUE (session_pk, turn_key) ); CREATE INDEX agent_turns_user_session_idx @@ -95,6 +98,12 @@ CREATE TABLE llm_usage_events ( reasoning_tokens bigint NOT NULL DEFAULT 0 CHECK (reasoning_tokens >= 0), total_tokens bigint NOT NULL DEFAULT 0 CHECK (total_tokens >= 0), observed_at timestamptz NOT NULL, + -- Agent-declared thread of the exchange; equal values continue one conversation. + stream_id text, + -- Provider identity of the response the exchange produced. + response_id text, + -- Response whose conversation the exchange's request continues. + previous_response_id text, metadata jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL DEFAULT now() ); @@ -186,3 +195,19 @@ CREATE TABLE search_outbox ( CREATE INDEX search_outbox_pending_idx ON search_outbox (available_at, id) WHERE processed_at IS NULL AND dead_lettered_at IS NULL; + + +-- Session names read back from the Agents' own title-generation calls, kept +-- apart from agent_sessions so session rows keep their shape. purpose marks +-- sessions that exist only to generate a title, such as Codex naming threads, +-- and parent_session_pk the task session such a thread was matched to. +CREATE TABLE session_titles ( + session_pk uuid PRIMARY KEY REFERENCES agent_sessions(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE, + title text, + title_event_pk uuid REFERENCES llm_usage_events(id) ON DELETE SET NULL, + title_observed_at timestamptz, + purpose text, + parent_session_pk uuid REFERENCES agent_sessions(id) ON DELETE SET NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/crates/abyss-backend/src/api.rs b/crates/abyss-backend/src/api.rs index 35a6629..8e1b355 100644 --- a/crates/abyss-backend/src/api.rs +++ b/crates/abyss-backend/src/api.rs @@ -7,11 +7,14 @@ use axum::{ Json, Router, body::Bytes, - extract::{DefaultBodyLimit, Path, Query, State}, + extract::{DefaultBodyLimit, Path, State}, http::{HeaderMap, HeaderValue, header}, response::{IntoResponse, Response}, routing::{get, post}, }; +// Filter keys repeat to select several values, e.g. `llm_model=a&llm_model=b`; +// axum's own Query cannot collect repeated keys. +use axum_extra::extract::Query; use serde::Serialize; use tower_http::trace::TraceLayer; use uuid::Uuid; @@ -22,12 +25,15 @@ use crate::{ search::{SessionSearchQuery, SessionSearchResponse}, storage::StorageBackend, usage::{ - IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryFields, SummaryQuery, TokenUsageSummaryResponse, + IngestEventsRequest, IngestEventsResponse, RawEventListResponse, RawEventsQuery, + SessionOutlineResponse, SessionTimelineResponse, SummaryFields, SummaryQuery, + TokenUsageSummaryResponse, UsageEventResponse, + session_list::{SessionListQuery, SessionListResponse}, + trajectory::{self, SessionTrajectoryResponse}, }, }; -const MAX_INGEST_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024; +const MAX_INGEST_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024; #[derive(Clone)] /// Cloneable dependencies and request limits shared by all Axum handlers. @@ -61,16 +67,26 @@ pub fn router(state: AppState) -> Router { .get(raw_events) .layer(DefaultBodyLimit::max(MAX_INGEST_REQUEST_BODY_BYTES)), ) + .route("/v1/agent-usage/events/{event_pk}", get(usage_event)) .route( "/v1/agent-usage/attachments/{attachment_id}", get(image_attachment), ) .route("/v1/agent-usage/summary", get(usage_summary)) .route("/v1/agent-usage/search", get(session_search)) + .route("/v1/agent-usage/sessions", get(list_sessions)) .route( "/v1/agent-usage/sessions/{session_pk}", get(session_timeline), ) + .route( + "/v1/agent-usage/sessions/{session_pk}/outline", + get(session_outline), + ) + .route( + "/v1/agent-usage/sessions/{session_pk}/trajectory", + get(session_trajectory), + ) .layer(TraceLayer::new_for_http()) .with_state(state) } @@ -142,6 +158,16 @@ async fn session_search( Ok(Json(state.storage.session_search(user_id, query).await?)) } +async fn list_sessions( + State(state): State, + headers: HeaderMap, + Query(query): Query, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + let query = query.validate()?; + Ok(Json(state.storage.list_sessions(user_id, query).await?)) +} + async fn session_timeline( State(state): State, headers: HeaderMap, @@ -152,6 +178,36 @@ async fn session_timeline( Ok(Json(response)) } +async fn session_outline( + State(state): State, + headers: HeaderMap, + Path(session_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + Ok(Json( + state.storage.session_outline(user_id, session_pk).await?, + )) +} + +async fn session_trajectory( + State(state): State, + headers: HeaderMap, + Path(session_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + let timeline = state.storage.session_timeline(user_id, session_pk).await?; + Ok(Json(trajectory::build(timeline))) +} + +async fn usage_event( + State(state): State, + headers: HeaderMap, + Path(event_pk): Path, +) -> Result, AppError> { + let user_id = state.identity.authenticate(&headers)?; + Ok(Json(state.storage.usage_event(user_id, event_pk).await?)) +} + async fn image_attachment( State(state): State, headers: HeaderMap, @@ -205,14 +261,18 @@ async fn raw_events( State(state): State, headers: HeaderMap, Query(query): Query, -) -> Result, AppError> { +) -> Result { let user_id = state.identity.authenticate(&headers)?; let default_page_size = state.default_page_size; + let includes_content = query.includes_content(); let response = state .storage .raw_events(user_id, query, default_page_size) .await?; - Ok(Json(response)) + if !includes_content { + return Ok(Json(RawEventListResponse::from_events(response)).into_response()); + } + Ok(Json(response).into_response()) } #[derive(Serialize)] diff --git a/crates/abyss-backend/src/db/mod.rs b/crates/abyss-backend/src/db/mod.rs index 0196460..1507008 100644 --- a/crates/abyss-backend/src/db/mod.rs +++ b/crates/abyss-backend/src/db/mod.rs @@ -9,7 +9,7 @@ pub mod models; /// Diesel's compile-time representation of the PostgreSQL schema. pub mod schema; -use diesel::{PgConnection, RunQueryDsl, r2d2::ConnectionManager, sql_query}; +use diesel::{IntoSql, PgConnection, RunQueryDsl, r2d2::ConnectionManager, sql_types::Integer}; use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations}; use crate::{config::Config, error::AppError}; @@ -37,6 +37,6 @@ pub fn run_migrations(pool: &DbPool) -> Result<(), Box Result<(), AppError> { - sql_query("SELECT 1").execute(connection)?; + diesel::select(1_i32.into_sql::()).execute(connection)?; Ok(()) } diff --git a/crates/abyss-backend/src/db/models.rs b/crates/abyss-backend/src/db/models.rs index 9e3b48a..74cb971 100644 --- a/crates/abyss-backend/src/db/models.rs +++ b/crates/abyss-backend/src/db/models.rs @@ -74,7 +74,7 @@ pub struct AgentSession { pub user_id: Uuid, /// Most recently associated device-context key. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently observed Agent version. pub agent_version: Option, @@ -102,7 +102,7 @@ pub struct NewAgentSession { pub user_id: Uuid, /// Associated device-context key. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Optional Agent version. pub agent_version: Option, @@ -131,8 +131,11 @@ pub struct AgentTurn { pub user_id: Uuid, /// Parent session primary key. pub session_pk: Uuid, - /// Positive turn number unique within the session. + /// Position of the turn in its session by start time. pub turn_index: i32, + /// Turn the Agent named for its requests; `None` for the turn opened by + /// requests naming none. + pub turn_key: Option, /// Earliest event observation in the turn. pub started_at: DateTime, /// Latest event observation in the turn. @@ -153,8 +156,11 @@ pub struct NewAgentTurn { pub user_id: Uuid, /// Parent session primary key. pub session_pk: Uuid, - /// Positive turn number unique within the session. + /// Position of the turn in its session by start time. pub turn_index: i32, + /// Turn the Agent named for its requests; `None` for the turn opened by + /// requests naming none. + pub turn_key: Option, /// Initial earliest observation time. pub started_at: DateTime, /// Initial latest observation time. @@ -182,7 +188,7 @@ pub struct UsageEvent { pub turn_pk: Uuid, /// Collector-generated idempotency key. pub event_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Agent version observed for the event. pub agent_version: Option, @@ -190,7 +196,7 @@ pub struct UsageEvent { pub session_id: String, /// Normalized turn number copied for direct filtering. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -218,6 +224,12 @@ pub struct UsageEvent { pub metadata: Value, /// Server insertion time. pub created_at: DateTime, + /// Agent-declared thread of the exchange. + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + pub previous_response_id: Option, } #[derive(Queryable, Selectable)] @@ -297,7 +309,7 @@ pub struct NewUsageEvent { pub turn_pk: Uuid, /// Collector-generated idempotency key. pub event_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Optional Agent version. pub agent_version: Option, @@ -305,7 +317,7 @@ pub struct NewUsageEvent { pub session_id: String, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -333,6 +345,12 @@ pub struct NewUsageEvent { pub metadata: Value, /// Server insertion time. pub created_at: DateTime, + /// Agent-declared thread of the exchange. + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + pub previous_response_id: Option, } #[derive(Insertable)] diff --git a/crates/abyss-backend/src/db/schema.rs b/crates/abyss-backend/src/db/schema.rs index 2e40b28..5cbd236 100644 --- a/crates/abyss-backend/src/db/schema.rs +++ b/crates/abyss-backend/src/db/schema.rs @@ -46,13 +46,14 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::{Integer, Nullable, Timestamptz, Uuid}; + use diesel::sql_types::{Integer, Nullable, Text, Timestamptz, Uuid}; agent_turns (id) { id -> Uuid, user_id -> Uuid, session_pk -> Uuid, turn_index -> Integer, + turn_key -> Nullable, started_at -> Timestamptz, ended_at -> Nullable, created_at -> Timestamptz, @@ -88,6 +89,9 @@ diesel::table! { observed_at -> Timestamptz, metadata -> Jsonb, created_at -> Timestamptz, + stream_id -> Nullable, + response_id -> Nullable, + previous_response_id -> Nullable, } } @@ -152,6 +156,21 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::{Nullable, Text, Timestamptz, Uuid}; + + session_titles (session_pk) { + session_pk -> Uuid, + user_id -> Uuid, + title -> Nullable, + title_event_pk -> Nullable, + title_observed_at -> Nullable, + purpose -> Nullable, + updated_at -> Timestamptz, + parent_session_pk -> Nullable, + } +} + diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -168,6 +187,7 @@ diesel::joinable!(agent_diagnostic_capture_events -> llm_usage_events (event_pk) diesel::joinable!(agent_diagnostic_captures -> agent_sessions (session_pk)); diesel::joinable!(agent_diagnostic_captures -> app_users (user_id)); diesel::joinable!(agent_diagnostic_captures -> devices (device_context_id)); +diesel::joinable!(session_titles -> agent_sessions (session_pk)); diesel::allow_tables_to_appear_in_same_query!( agent_diagnostic_capture_events, @@ -179,4 +199,5 @@ diesel::allow_tables_to_appear_in_same_query!( llm_usage_events, llm_usage_event_attachments, search_outbox, + session_titles, ); diff --git a/crates/abyss-backend/src/search/mod.rs b/crates/abyss-backend/src/search/mod.rs index c2e9164..fa04921 100644 --- a/crates/abyss-backend/src/search/mod.rs +++ b/crates/abyss-backend/src/search/mod.rs @@ -32,12 +32,15 @@ pub struct SessionSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Optional canonical Agent filter. - pub agent_name: Option, - /// Optional canonical LLM provider filter. - pub llm_provider: Option, - /// Optional model filter. - pub llm_model: Option, + /// Optional Agent filter, matched exactly; repeat the key to match any of several. + #[serde(default)] + pub agent_name: Vec, + /// Optional LLM provider filter, matched exactly; repeat the key to match any of several. + #[serde(default)] + pub llm_provider: Vec, + /// Optional model filter; repeat the key to match any of several. + #[serde(default)] + pub llm_model: Vec, /// Optional request or response filter. pub event_type: Option, /// One-based result page. @@ -93,11 +96,9 @@ impl SessionSearchQuery { text, from: self.from, to: self.to, - agent_name: normalized_optional(self.agent_name.as_deref(), "agent_name")? - .map(|value| canonical_agent_name(&value)), - llm_provider: normalized_optional(self.llm_provider.as_deref(), "llm_provider")? - .map(|value| normalize_slug(&value)), - llm_model: normalized_optional(self.llm_model.as_deref(), "llm_model")?, + agent_names: normalized_list(&self.agent_name, "agent_name")?, + llm_providers: normalized_list(&self.llm_provider, "llm_provider")?, + llm_models: normalized_list(&self.llm_model, "llm_model")?, event_type, page, page_size, @@ -114,12 +115,12 @@ pub struct ValidatedSearchQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Canonical Agent-name filter. - pub agent_name: Option, - /// Canonical provider slug filter. - pub llm_provider: Option, - /// Trimmed model filter. - pub llm_model: Option, + /// Trimmed Agent names to match exactly, any of, or empty for every Agent. + pub agent_names: Vec, + /// Trimmed providers to match any of, or empty for every provider. + pub llm_providers: Vec, + /// Trimmed models to match any of, or empty for every model. + pub llm_models: Vec, /// Canonical request or response filter. pub event_type: Option, /// One-based result page. @@ -154,7 +155,7 @@ pub struct SessionSearchResult { pub session_pk: Uuid, /// Agent-native session identifier. pub session_id: String, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently observed Agent version. pub agent_version: Option, @@ -187,7 +188,7 @@ pub struct SessionSearchMatch { pub turn_index: i32, /// Request or response side. pub event_type: String, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -270,24 +271,15 @@ fn normalized_optional(value: Option<&str>, field: &str) -> Result String { - value - .trim() - .to_ascii_lowercase() - .chars() - .map(|character| match character { - '_' | ' ' => '-', - other => other, - }) - .collect() -} - -fn canonical_agent_name(value: &str) -> String { - let normalized = normalize_slug(value); - match normalized.as_str() { - "claude" | "claude-desktop" => "claude-code".to_owned(), - _ => normalized, - } +/// Validates and trims every value, then de-duplicates them. +fn normalized_list(values: &[String], field: &str) -> Result, AppError> { + let mut normalized = values + .iter() + .map(|value| normalized_required(value, field, MAX_FILTER_CHARACTERS)) + .collect::, _>>()?; + normalized.sort_unstable(); + normalized.dedup(); + Ok(normalized) } #[cfg(test)] @@ -322,6 +314,18 @@ mod tests { assert!(error.to_string().contains("first 10000 matches")); } + #[test] + fn matches_agent_names_exactly_as_reported() { + let mut agents = query("hello"); + agents.agent_name = vec![ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "claude-code".to_owned(), + ]; + let validated = agents.validate().expect("agent filter is valid"); + assert_eq!(validated.agent_names, ["claude-code", "claude-desktop"]); + } + #[test] fn parses_highlights_without_rendering_html() { let fragment = SearchFragment::parse(&format!( @@ -338,9 +342,9 @@ mod tests { q: q.to_owned(), from: None, to: None, - agent_name: None, - llm_provider: None, - llm_model: None, + agent_name: Vec::new(), + llm_provider: Vec::new(), + llm_model: Vec::new(), event_type: None, page: None, page_size: None, diff --git a/crates/abyss-backend/src/search/projection.rs b/crates/abyss-backend/src/search/projection.rs index 4bba669..c4e4733 100644 --- a/crates/abyss-backend/src/search/projection.rs +++ b/crates/abyss-backend/src/search/projection.rs @@ -7,15 +7,12 @@ use super::{HIGHLIGHT_END, HIGHLIGHT_START}; pub const MAX_CONTENT_CHARACTERS: usize = 1_000_000; const MAX_METADATA_FIELD_CHARACTERS: usize = 100_000; const MAX_EXTRACTED_VALUES: usize = 64; -const MAX_JSON_DEPTH: usize = 8; /// Searchable values derived from one source usage event. pub struct SearchProjection { pub content: Option, pub tool_names: Vec, pub tool_content: Vec, - pub commands: Vec, - pub file_paths: Vec, } impl SearchProjection { @@ -27,8 +24,6 @@ impl SearchProjection { .map(|value| truncate_characters(value, MAX_CONTENT_CHARACTERS)), tool_names: extracted.tool_names, tool_content: extracted.tool_content, - commands: extracted.commands, - file_paths: extracted.file_paths, } } } @@ -36,8 +31,6 @@ impl SearchProjection { struct SearchableMetadata { tool_names: Vec, tool_content: Vec, - commands: Vec, - file_paths: Vec, } impl SearchableMetadata { @@ -45,14 +38,8 @@ impl SearchableMetadata { let mut extracted = Self { tool_names: Vec::new(), tool_content: Vec::new(), - commands: Vec::new(), - file_paths: Vec::new(), }; - if let Some(working_directory) = metadata.get("working_directory").and_then(Value::as_str) { - extracted.push_file_path(working_directory); - } - // Ignore every other top-level key so headers, credentials, and image // data never enter a derived full-text index. let Some(segments) = metadata.get("content_segments").and_then(Value::as_array) else { @@ -69,9 +56,6 @@ impl SearchableMetadata { } if let Some(input) = object.get("input").and_then(Value::as_str) { push_bounded(&mut extracted.tool_content, input); - if let Ok(value) = serde_json::from_str::(input) { - extracted.extract_structured_input(&value, MAX_JSON_DEPTH); - } } } Some("tool_result") => { @@ -84,36 +68,6 @@ impl SearchableMetadata { } extracted } - - fn extract_structured_input(&mut self, value: &Value, remaining_depth: usize) { - if remaining_depth == 0 { - return; - } - match value { - Value::Object(object) => { - for (key, child) in object.iter().take(MAX_EXTRACTED_VALUES) { - if let Some(text) = child.as_str() { - match key.as_str() { - "command" | "cmd" => push_bounded(&mut self.commands, text), - "file" | "file_path" | "path" => self.push_file_path(text), - _ => {} - } - } - self.extract_structured_input(child, remaining_depth.saturating_sub(1)); - } - } - Value::Array(items) => { - for item in items.iter().take(MAX_EXTRACTED_VALUES) { - self.extract_structured_input(item, remaining_depth.saturating_sub(1)); - } - } - _ => {} - } - } - - fn push_file_path(&mut self, value: &str) { - push_bounded(&mut self.file_paths, value); - } } fn push_bounded(values: &mut Vec, value: &str) { diff --git a/crates/abyss-backend/src/storage/mod.rs b/crates/abyss-backend/src/storage/mod.rs index 7cb6aa6..3f846cc 100644 --- a/crates/abyss-backend/src/storage/mod.rs +++ b/crates/abyss-backend/src/storage/mod.rs @@ -20,7 +20,10 @@ use crate::{ search::{SessionSearchQuery, SessionSearchResponse}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -48,6 +51,13 @@ pub trait StorageBackend: Send + Sync { default_page_size: i64, ) -> BoxedFuture<'_, Result>; + /// Loads one owner-scoped request or response event with its full content. + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result>; + /// Aggregates owner-scoped event and token usage. fn usage_summary( &self, @@ -63,6 +73,20 @@ pub trait StorageBackend: Send + Sync { session_pk: Uuid, ) -> BoxedFuture<'_, Result>; + /// Loads one owner-scoped session timeline without event content bodies. + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result>; + + /// Lists one page of owner-scoped sessions, most recently active first. + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result>; + /// Loads authorized image bytes by attachment identifier. fn image_attachment( &self, diff --git a/crates/abyss-backend/src/storage/postgres.rs b/crates/abyss-backend/src/storage/postgres.rs index e9716cd..70b592d 100644 --- a/crates/abyss-backend/src/storage/postgres.rs +++ b/crates/abyss-backend/src/storage/postgres.rs @@ -2,6 +2,8 @@ mod repository; mod search; +mod session_list; +mod session_titles; use std::time::Duration; @@ -17,7 +19,10 @@ use crate::{ storage::{BoxedFuture, StorageBackend}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -113,6 +118,17 @@ impl StorageBackend for PostgresEsBackend { }) } + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| repository::usage_event(connection, user_id, event_pk)) + .await + }) + } + fn usage_summary( &self, user_id: Uuid, @@ -140,6 +156,30 @@ impl StorageBackend for PostgresEsBackend { }) } + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| { + repository::session_outline(connection, user_id, session_pk) + }) + .await + }) + } + + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| session_list::list_sessions(connection, user_id, &query)) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/postgres/repository.rs b/crates/abyss-backend/src/storage/postgres/repository.rs index df586b8..a63169e 100644 --- a/crates/abyss-backend/src/storage/postgres/repository.rs +++ b/crates/abyss-backend/src/storage/postgres/repository.rs @@ -10,13 +10,16 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - Connection, ExpressionMethods, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, - dsl::{max, sql}, + Connection, ExpressionMethods, IntoSql, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, + SelectableHelper, define_sql_function, + dsl::{count_star, max, sql}, result::OptionalExtension, sql_query, sql_types::{ - Array, BigInt, Bool, Integer, Jsonb, Nullable, Text, Timestamptz, Uuid as SqlUuid, + Array, BigInt, Bool, Bytea, Integer, Jsonb, Nullable, SingleValue, Text, Timestamptz, + Uuid as SqlUuid, }, + upsert::excluded, }; use uuid::Uuid; @@ -35,9 +38,10 @@ use crate::{ }, error::AppError, usage::{ - IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, RawEventsQuery, - RawEventsResponse, SessionInfo, SessionTimelineResponse, SummaryQuery, SummaryResponse, - SummaryRow, TurnTimeline, UsageEventResponse, + IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, OUTLINE_PREVIEW_CHARS, + RawEventsQuery, RawEventsResponse, SessionInfo, SessionOutlineResponse, + SessionTimelineResponse, SummaryQuery, SummaryResponse, SummaryRow, TurnTimeline, + UsageEventOutline, UsageEventResponse, attachments::{ ImageMediaType, StoredImageAttachment, UsageEventAttachmentResponse, ValidatedImageAttachment, @@ -45,17 +49,18 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, TurnIdentityKind, agent_name_filter_values, canonical_agent_name, - choose_turn_index, escape_like_pattern, non_empty, non_negative_count, normalize_limit, - normalize_slug, normalize_text, normalized_optional, normalized_tokens, parse_group_by, - sha256_bytes, turn_identity, unix_epoch, validate_batch, validate_event_type, + escape_like_pattern, exact_values_filter, non_empty, non_negative_count, + normalize_limit, normalize_text, parse_group_by, sha256_bytes, unix_epoch, + validate_batch, validate_event_type, }, - session_metadata_from_event, }, }; -#[cfg(test)] -use crate::usage::persistence::{turn_identity_from_metadata, validate_event}; +use super::session_titles; + +define_sql_function!(fn coalesce(value: Nullable, fallback: Nullable) -> Nullable); +define_sql_function!(fn least(left: ST, right: ST) -> ST); +define_sql_function!(fn greatest(left: ST, right: ST) -> ST); const SUMMARY_AGGREGATE_SQL: &str = "\ SELECT \ @@ -66,12 +71,7 @@ const SUMMARY_AGGREGATE_SQL: &str = "\ CASE WHEN $3 THEN d.host_name END AS host_name, \ CASE WHEN $3 THEN d.platform END AS platform, \ CASE WHEN $3 THEN d.os_version END AS os_version, \ - CASE \ - WHEN $4 THEN CASE \ - WHEN e.agent_name IN ('claude', 'claude-code', 'claude-desktop') THEN 'claude-code' \ - ELSE e.agent_name \ - END \ - END AS agent_name, \ + CASE WHEN $4 THEN e.agent_name END AS agent_name, \ CASE WHEN $5 THEN e.llm_provider END AS llm_provider, \ CASE WHEN $6 THEN e.llm_model END AS llm_model, \ CASE WHEN $7 THEN e.event_type END AS event_type, \ @@ -100,11 +100,11 @@ const SUMMARY_AGGREGATE_SQL: &str = "\ )) \ ) \ AND ($17 = false OR e.agent_name = ANY($18)) \ - AND ($19 = false OR e.session_id = $20) \ + AND ($19 = false OR e.session_id = ANY($20)) \ AND ($21 = false OR e.session_pk = $22) \ AND ($23 = false OR e.turn_index = $24) \ - AND ($25 = false OR e.llm_provider = $26) \ - AND ($27 = false OR e.llm_model = $28) \ + AND ($25 = false OR e.llm_provider = ANY($26)) \ + AND ($27 = false OR e.llm_model = ANY($28)) \ AND ($29 = false OR e.event_type = $30) \ GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 \ ORDER BY total_tokens DESC, agent_name ASC NULLS LAST, user_email ASC NULLS LAST, day ASC NULLS LAST \ @@ -248,12 +248,12 @@ pub fn usage_summary( from: query.from, to: query.to, user_filter: query.user_filter.as_deref(), - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: None, turn_index: None, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit, offset: 0, @@ -288,16 +288,16 @@ fn load_filtered_summary_rows( let has_user_filter = !user_filter_pattern.is_empty(); let agent_names = sql_filters.agent_names; let has_agent_filter = !agent_names.is_empty(); - let session_id = sql_filters.session_id.unwrap_or_default(); - let has_session_id_filter = !session_id.is_empty(); + let session_ids = sql_filters.session_ids; + let has_session_id_filter = !session_ids.is_empty(); let session_pk = sql_filters.session_pk.unwrap_or_else(Uuid::nil); let has_session_pk_filter = session_pk != Uuid::nil(); let turn_index = sql_filters.turn_index.unwrap_or_default(); let has_turn_index_filter = turn_index > 0_i32; - let provider = sql_filters.provider.unwrap_or_default(); - let has_provider_filter = !provider.is_empty(); - let model = sql_filters.model.unwrap_or_default(); - let has_model_filter = !model.is_empty(); + let providers = sql_filters.providers; + let has_provider_filter = !providers.is_empty(); + let models = sql_filters.models; + let has_model_filter = !models.is_empty(); let event_type = sql_filters.event_type.unwrap_or_default(); let has_event_type_filter = !event_type.is_empty(); @@ -324,15 +324,15 @@ fn load_filtered_summary_rows( .bind::(has_agent_filter) .bind::, _>(agent_names) .bind::(has_session_id_filter) - .bind::(&session_id) + .bind::, _>(session_ids) .bind::(has_session_pk_filter) .bind::(session_pk) .bind::(has_turn_index_filter) .bind::(turn_index) .bind::(has_provider_filter) - .bind::(&provider) + .bind::, _>(providers) .bind::(has_model_filter) - .bind::(&model) + .bind::, _>(models) .bind::(has_event_type_filter) .bind::(&event_type) .bind::(filters.limit) @@ -347,20 +347,7 @@ pub fn session_timeline( user_id: Uuid, session_pk: Uuid, ) -> Result { - // Timeline views include prompt/response text, so they are always limited - // to the owner. - let session = agent_sessions::table - .filter(agent_sessions::id.eq(session_pk)) - .filter(agent_sessions::user_id.eq(user_id)) - .first::(connection) - .optional()? - .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))?; - touch_session_metadata(&session); - - let turns = agent_turns::table - .filter(agent_turns::session_pk.eq(session.id)) - .order(agent_turns::turn_index.asc()) - .load::(connection)?; + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; let mut events = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(session.id)) @@ -386,7 +373,130 @@ pub fn session_timeline( attachments_by_event.remove(&event_pk).unwrap_or_default(), )); } + let mut timeline = assemble_timeline(session, turns, events_by_turn); + session_titles::fill(connection, &mut timeline.session)?; + Ok(timeline) +} +type OutlineRow = ( + Uuid, + String, + Uuid, + i32, + String, + String, + String, + i64, + i64, + i64, + i64, + i64, + i64, + DateTime, + i64, + Option, +); + +/// Loads one owner-scoped session timeline without event content bodies. +pub fn session_outline( + connection: &mut PgConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result { + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; + // Only fixed-size columns and SQL-computed summaries are selected, so the + // outline stays small no matter how large event text or metadata grows. + let rows = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(session.id)) + .select(( + llm_usage_events::id, + llm_usage_events::event_id, + llm_usage_events::turn_pk, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + sql::("coalesce(char_length(text), 0)::bigint"), + sql::>(&format!( + "CASE WHEN event_type = 'request' THEN right(text, {OUTLINE_PREVIEW_CHARS}) \ + ELSE left(text, {OUTLINE_PREVIEW_CHARS}) END" + )), + )) + .load::(connection)?; + let event_ids = rows.iter().map(|row| row.0).collect::>(); + let attachment_counts = llm_usage_event_attachments::table + .filter(llm_usage_event_attachments::event_pk.eq_any(event_ids)) + .group_by(llm_usage_event_attachments::event_pk) + .select((llm_usage_event_attachments::event_pk, count_star())) + .load::<(Uuid, i64)>(connection)? + .into_iter() + .collect::>(); + let mut events = rows + .into_iter() + .map(|row| UsageEventOutline { + id: row.0, + event_id: row.1, + turn_pk: row.2, + turn_index: row.3, + llm_provider: row.4, + llm_model: row.5, + event_type: row.6, + input_tokens: row.7, + output_tokens: row.8, + cache_read_tokens: row.9, + cache_write_tokens: row.10, + reasoning_tokens: row.11, + total_tokens: row.12, + observed_at: row.13, + text_chars: row.14, + preview: row.15, + attachment_count: attachment_counts.get(&row.0).copied().unwrap_or(0), + }) + .collect::>(); + UsageEventTimelineOrder::sort(&mut events); + let mut events_by_turn: HashMap> = HashMap::new(); + for event in events { + events_by_turn.entry(event.turn_pk).or_default().push(event); + } + let mut timeline = assemble_timeline(session, turns, events_by_turn); + session_titles::fill(connection, &mut timeline.session)?; + Ok(timeline) +} + +fn load_session_turns( + connection: &mut PgConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result<(AgentSession, Vec), AppError> { + // Timeline views include prompt/response text, so they are always limited + // to the owner. + let session = agent_sessions::table + .filter(agent_sessions::id.eq(session_pk)) + .filter(agent_sessions::user_id.eq(user_id)) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))?; + touch_session_metadata(&session); + + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session.id)) + .order(agent_turns::turn_index.asc()) + .load::(connection)?; + Ok((session, turns)) +} + +fn assemble_timeline( + session: AgentSession, + turns: Vec, + mut events_by_turn: HashMap>, +) -> SessionTimelineResponse { let turn_timelines = turns .into_iter() .map(|turn| { @@ -401,20 +511,23 @@ pub fn session_timeline( }) .collect(); - Ok(SessionTimelineResponse { + SessionTimelineResponse { session: SessionInfo { session_pk: session.id, user_id: session.user_id, device_context_id: session.device_context_id, - agent_name: canonical_agent_name(&session.agent_name), + agent_name: session.agent_name, agent_version: session.agent_version, session_id: session.session_id, started_at: session.started_at, ended_at: session.ended_at, metadata: session.metadata, + title: None, + purpose: None, + parent_session_pk: None, }, turns: turn_timelines, - }) + } } /// Returns one newest-first page of raw owner-scoped events. @@ -437,16 +550,17 @@ pub fn raw_events( from: query.from, to: query.to, user_filter: None, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: query.session_pk, turn_index: query.turn_index, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit: page_size.saturating_add(1_i64), offset, }, + query.includes_content(), )?; // Fetch one sentinel row beyond the requested page so no count query is // needed merely to determine whether a next offset exists. @@ -455,7 +569,11 @@ pub fn raw_events( events.truncate(page_size_usize); } let device_contexts = load_device_contexts_for_events(connection, &events)?; - let mut attachments_by_event = load_attachments_for_events(connection, &events)?; + let mut attachments_by_event = if query.includes_content() { + load_attachments_for_events(connection, &events)? + } else { + HashMap::new() + }; Ok(RawEventsResponse { events: events @@ -474,6 +592,33 @@ pub fn raw_events( }) } +/// Loads one owner-scoped event with the same shape used by timelines. +pub fn usage_event( + connection: &mut PgConnection, + user_id: Uuid, + event_pk: Uuid, +) -> Result { + // Event content can contain sensitive conversation context. Keep this + // endpoint scoped to the deployment owner. + let event = llm_usage_events::table + .filter(llm_usage_events::id.eq(event_pk)) + .filter(llm_usage_events::user_id.eq(user_id)) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("usage event not found: {event_pk}")))?; + let events = [event]; + let device_contexts = load_device_contexts_for_events(connection, &events)?; + let mut attachments_by_event = load_attachments_for_events(connection, &events)?; + let [event] = events; + let event_pk = event.id; + let device_context_id = event.device_context_id; + Ok(event_response( + event, + device_contexts.get(&device_context_id), + attachments_by_event.remove(&event_pk).unwrap_or_default(), + )) +} + /// Loads authorized attachment bytes and verifies persisted size metadata. pub fn image_attachment( connection: &mut PgConnection, @@ -522,10 +667,10 @@ fn ingest_one_event( ) -> Result { let observed_at = event.observed_at; let now = Utc::now(); - let agent_name = canonical_agent_name(&event.agent.name); - let agent_version = normalized_optional(event.agent.version.as_deref()); - let provider = normalize_slug(&event.llm.provider); - let model = normalize_text(&event.llm.model); + let agent_name = event.agent.name.clone(); + let agent_version = event.agent.version.clone(); + let provider = event.llm.provider.clone(); + let model = event.llm.model.clone(); let device = upsert_device(connection, user_id, event, observed_at, now)?; let session = upsert_session( @@ -540,17 +685,16 @@ fn ingest_one_event( now, }, )?; - let turn_index = resolve_turn_index(connection, session.id, event)?; - let turn = upsert_turn( + let (turn_pk, turn_index) = place_in_turn( connection, user_id, session.id, - turn_index, + event.turn_id.as_deref(), observed_at, now, )?; - let tokens = normalized_tokens(event)?; - let text = normalized_optional(event.text.as_deref()); + let tokens = &event.token_usage; + let text = event.text.clone(); // `event_id` is the idempotency key supplied by the collector. Duplicate // observations are accepted as no-ops instead of failing the whole batch. @@ -561,26 +705,29 @@ fn ingest_one_event( user_id, device_context_id: device.id, session_pk: session.id, - turn_pk: turn.id, - event_id: normalize_text(&event.event_id), - agent_name, + turn_pk, + event_id: event.event_id.clone(), + agent_name: agent_name.clone(), agent_version, - session_id: normalize_text(&event.session_id), + session_id: event.session_id.clone(), turn_index, llm_provider: provider, llm_model: model, event_type: event.event_type.as_str().to_owned(), text_sha256: text.as_deref().map(sha256_bytes), text, - input_tokens: tokens.input, - output_tokens: tokens.output, - cache_read_tokens: tokens.cache_read, - cache_write_tokens: tokens.cache_write, - reasoning_tokens: tokens.reasoning, - total_tokens: tokens.total, + input_tokens: tokens.input_tokens.unwrap_or_default(), + output_tokens: tokens.output_tokens.unwrap_or_default(), + cache_read_tokens: tokens.cache_read_tokens.unwrap_or_default(), + cache_write_tokens: tokens.cache_write_tokens.unwrap_or_default(), + reasoning_tokens: tokens.reasoning_tokens.unwrap_or_default(), + total_tokens: tokens.total_tokens.unwrap_or_default(), observed_at, metadata: event.metadata.clone(), created_at: now, + stream_id: event.stream_id.clone(), + response_id: event.response_id.clone(), + previous_response_id: event.previous_response_id.clone(), }) .on_conflict(llm_usage_events::event_id) .do_nothing() @@ -589,23 +736,7 @@ fn ingest_one_event( // Child rows are written only for a newly inserted event. A replay must not // replace the attachments associated with the original idempotency key. if inserted == 1 && !attachments.is_empty() { - let attachment_rows = attachments - .into_iter() - .map(|attachment| NewUsageEventAttachment { - id: Uuid::now_v7(), - user_id, - event_pk: usage_event_id, - position: attachment.position, - media_type: attachment.media_type.as_str().to_owned(), - byte_size: attachment.byte_size, - sha256: attachment.sha256, - content: attachment.content, - created_at: now, - }) - .collect::>(); - diesel::insert_into(llm_usage_event_attachments::table) - .values(&attachment_rows) - .execute(connection)?; + insert_attachments(connection, user_id, usage_event_id, attachments, now)?; } // The enclosing transaction commits the event and its projection task @@ -618,11 +749,50 @@ fn ingest_one_event( created_at: now, }) .execute(connection)?; + session_titles::capture( + connection, + &session_titles::CapturedEvent { + user_id, + session_pk: session.id, + event_pk: usage_event_id, + event_type: event.event_type.as_str(), + metadata: &event.metadata, + naming_thread: event.naming_thread, + observed_at, + }, + )?; } Ok(inserted == 1) } +fn insert_attachments( + connection: &mut PgConnection, + user_id: Uuid, + event_pk: Uuid, + attachments: Vec, + now: DateTime, +) -> Result<(), AppError> { + let attachment_rows = attachments + .into_iter() + .map(|attachment| NewUsageEventAttachment { + id: Uuid::now_v7(), + user_id, + event_pk, + position: attachment.position, + media_type: attachment.media_type.as_str().to_owned(), + byte_size: attachment.byte_size, + sha256: attachment.sha256, + content: attachment.content, + created_at: now, + }) + .collect::>(); + diesel::insert_into(llm_usage_event_attachments::table) + .values(&attachment_rows) + .execute(connection)?; + Ok(()) +} + fn upsert_device( connection: &mut PgConnection, user_id: Uuid, @@ -637,9 +807,9 @@ fn upsert_device( .values(NewDevice { id: Uuid::now_v7(), user_id, - host_name: normalize_text(&event.device.host_name), - platform: normalize_slug(&event.device.platform), - os_version: normalized_optional(event.device.os_version.as_deref()), + host_name: event.device.host_name.clone(), + platform: event.device.platform.clone(), + os_version: event.device.os_version.clone(), first_seen_at: observed_at, last_seen_at: observed_at, created_at: now, @@ -648,14 +818,14 @@ fn upsert_device( .on_conflict((devices::user_id, devices::host_name, devices::platform)) .do_update() .set(( - devices::os_version.eq(sql::>( - "COALESCE(EXCLUDED.os_version, devices.os_version)", - )), - devices::first_seen_at.eq(sql::( - "LEAST(devices.first_seen_at, EXCLUDED.first_seen_at)", + devices::os_version.eq(coalesce(excluded(devices::os_version), devices::os_version)), + devices::first_seen_at.eq(least( + devices::first_seen_at, + excluded(devices::first_seen_at), )), - devices::last_seen_at.eq(sql::( - "GREATEST(devices.last_seen_at, EXCLUDED.last_seen_at)", + devices::last_seen_at.eq(greatest( + devices::last_seen_at, + excluded(devices::last_seen_at), )), devices::updated_at.eq(now), )) @@ -667,7 +837,7 @@ fn upsert_session( connection: &mut PgConnection, input: UpsertSessionInput<'_>, ) -> Result { - let session_id = normalize_text(&input.event.session_id); + let session_id = input.event.session_id.clone(); diesel::insert_into(agent_sessions::table) .values(NewAgentSession { id: Uuid::now_v7(), @@ -678,7 +848,7 @@ fn upsert_session( session_id, started_at: input.observed_at, ended_at: Some(input.observed_at), - metadata: session_metadata_from_event(&input.event.metadata), + metadata: serde_json::Value::Object(serde_json::Map::new()), created_at: input.now, updated_at: input.now, }) @@ -689,156 +859,188 @@ fn upsert_session( )) .do_update() .set(( - agent_sessions::device_context_id - .eq(sql::("EXCLUDED.device_context_id")), - agent_sessions::agent_version.eq(sql::>( - "COALESCE(EXCLUDED.agent_version, agent_sessions.agent_version)", + agent_sessions::device_context_id.eq(excluded(agent_sessions::device_context_id)), + agent_sessions::agent_version.eq(coalesce( + excluded(agent_sessions::agent_version), + agent_sessions::agent_version, )), - agent_sessions::started_at.eq(sql::( - "LEAST(agent_sessions.started_at, EXCLUDED.started_at)", + agent_sessions::started_at.eq(least( + agent_sessions::started_at, + excluded(agent_sessions::started_at), )), - agent_sessions::ended_at.eq(sql::>( - "GREATEST(COALESCE(agent_sessions.ended_at, EXCLUDED.ended_at), EXCLUDED.ended_at)", + agent_sessions::ended_at.eq(greatest( + coalesce(agent_sessions::ended_at, excluded(agent_sessions::ended_at)), + excluded(agent_sessions::ended_at), )), - agent_sessions::metadata - .eq(sql::("agent_sessions.metadata || EXCLUDED.metadata")), agent_sessions::updated_at.eq(input.now), )) .get_result::(connection) .map_err(AppError::from) } -fn resolve_turn_index( +/// Places an event in a turn of its session and returns the turn's id and +/// position. +/// +/// Events naming the same turn share it, and an event naming none joins the +/// turn in progress when it was observed. Turns are numbered by when they +/// started, so a turn that arrives late moves the turns after it. +fn place_in_turn( connection: &mut PgConnection, + user_id: Uuid, session_pk: Uuid, - event: &IngestUsageEvent, -) -> Result { - // Older collectors without a stable provider identity retain their supplied - // index. Newer evidence lets restarted collectors avoid merging new work - // into an already-used local turn number. - let Some(identity) = turn_identity(event) else { - return Ok(event.turn_index); + turn_id: Option<&str>, + observed_at: DateTime, + now: DateTime, +) -> Result<(Uuid, i32), AppError> { + let found = match turn_id { + Some(key) => named_turn(connection, session_pk, key)?, + None => turn_in_progress(connection, session_pk, observed_at)?, }; - let existing_turn_index = existing_turn_index_for_identity(connection, session_pk, &identity)?; - let next_turn_index = next_session_turn_index(connection, session_pk)?; - Ok(choose_turn_index( - event.turn_index, - existing_turn_index, - next_turn_index, - )) + let (turn_pk, turn_index, reordered) = if let Some(turn) = found { + diesel::update(agent_turns::table.find(turn.id)) + .set(( + agent_turns::started_at.eq(turn.started_at.min(observed_at)), + agent_turns::ended_at + .eq(Some(turn.ended_at.unwrap_or(observed_at).max(observed_at))), + agent_turns::updated_at.eq(now), + )) + .execute(connection)?; + (turn.id, turn.turn_index, observed_at < turn.started_at) + } else { + let last = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .select(max(agent_turns::turn_index)) + .get_result::>(connection)?; + let id = Uuid::now_v7(); + let turn_index = last.unwrap_or(0_i32).saturating_add(1_i32); + diesel::insert_into(agent_turns::table) + .values(NewAgentTurn { + id, + user_id, + session_pk, + turn_index, + turn_key: turn_id.map(str::to_owned), + started_at: observed_at, + ended_at: Some(observed_at), + created_at: now, + updated_at: now, + }) + .execute(connection)?; + (id, turn_index, last.is_some()) + }; + let turn_index = if reordered && number_turns(connection, session_pk)? { + agent_turns::table + .find(turn_pk) + .select(agent_turns::turn_index) + .get_result::(connection)? + } else { + turn_index + }; + Ok((turn_pk, turn_index)) } -fn existing_turn_index_for_identity( +/// The turn named `key` in a session. The first named turn adopts the turn +/// that events naming none opened at the start of the session. +fn named_turn( connection: &mut PgConnection, session_pk: Uuid, - identity: &TurnIdentity, -) -> Result, AppError> { - let row = match &identity.kind { - TurnIdentityKind::CodexTurnId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'codex_turn_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::ClaudeTurnId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'claude_turn_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::ResponseId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'response_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::MessageId => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'message_id' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), - TurnIdentityKind::RequestHash => sql_query( - "SELECT turn_index \ - FROM llm_usage_events \ - WHERE session_pk = $1 AND metadata->>'request_hash' = $2 \ - ORDER BY observed_at ASC \ - LIMIT 1", - ) - .bind::(session_pk) - .bind::(&identity.value) - .get_result::(connection), + key: &str, +) -> Result, AppError> { + let named = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .filter(agent_turns::turn_key.eq(key)) + .select(AgentTurn::as_select()) + .first::(connection) + .optional()?; + if named.is_some() { + return Ok(named); + } + let mut turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .select(AgentTurn::as_select()) + .limit(2_i64) + .load::(connection)?; + let opening = match turns.as_slice() { + [turn] if turn.turn_key.is_none() => turns.pop(), + _ => None, }; - row.optional() - .map(|row| row.map(|row| row.turn_index)) - .map_err(AppError::from) + if let Some(turn) = &opening { + diesel::update(agent_turns::table.find(turn.id)) + .set(agent_turns::turn_key.eq(key)) + .execute(connection)?; + } + Ok(opening) } -fn next_session_turn_index( +/// The turn in progress when an event naming no turn was observed: the latest +/// turn started by then, else the first turn of the session. +fn turn_in_progress( connection: &mut PgConnection, session_pk: Uuid, -) -> Result { + observed_at: DateTime, +) -> Result, AppError> { + let started = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .filter(agent_turns::started_at.le(observed_at)) + .order_by((agent_turns::started_at.desc(), agent_turns::id.desc())) + .select(AgentTurn::as_select()) + .first::(connection) + .optional()?; + if started.is_some() { + return Ok(started); + } agent_turns::table .filter(agent_turns::session_pk.eq(session_pk)) - .select(max(agent_turns::turn_index)) - .get_result::>(connection) - .map(|value| value.unwrap_or(0_i32).saturating_add(1_i32)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select(AgentTurn::as_select()) + .first::(connection) + .optional() .map_err(AppError::from) } -fn upsert_turn( - connection: &mut PgConnection, - user_id: Uuid, - session_pk: Uuid, - turn_index: i32, - observed_at: DateTime, - now: DateTime, -) -> Result { - diesel::insert_into(agent_turns::table) - .values(NewAgentTurn { - id: Uuid::now_v7(), - user_id, - session_pk, - turn_index, - started_at: observed_at, - ended_at: Some(observed_at), - created_at: now, - updated_at: now, - }) - .on_conflict((agent_turns::session_pk, agent_turns::turn_index)) - .do_update() - .set(( - agent_turns::started_at.eq(sql::( - "LEAST(agent_turns.started_at, EXCLUDED.started_at)", - )), - agent_turns::ended_at.eq(sql::>( - "GREATEST(COALESCE(agent_turns.ended_at, EXCLUDED.ended_at), EXCLUDED.ended_at)", - )), - agent_turns::updated_at.eq(now), - )) - .get_result::(connection) - .map_err(AppError::from) +/// Numbers the turns of a session, and their events, by when the turns +/// started. Writes nothing and returns `false` while the numbers already +/// follow that order. +fn number_turns(connection: &mut PgConnection, session_pk: Uuid) -> Result { + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session_pk)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select((agent_turns::id, agent_turns::turn_index)) + .load::<(Uuid, i32)>(connection)?; + let moved = turns + .iter() + .zip(1_i32..) + .filter(|((_, index), position)| index != position) + .map(|((id, _), position)| (*id, position)) + .collect::>(); + if moved.is_empty() { + return Ok(false); + } + // Moved turns first step above every current number, so no single-row + // update collides with another turn's number or goes below one. + let above = turns.iter().map(|(_, index)| *index).max().unwrap_or(0_i32); + for (id, position) in &moved { + diesel::update(agent_turns::table.find(*id)) + .set(agent_turns::turn_index.eq(above.saturating_add(*position))) + .execute(connection)?; + } + for (id, position) in moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(position)) + .execute(connection)?; + diesel::update(llm_usage_events::table.filter(llm_usage_events::turn_pk.eq(id))) + .set(llm_usage_events::turn_index.eq(position)) + .execute(connection)?; + } + Ok(true) } +/// Loads matching events newest first. Without content, text and metadata are +/// not read: they hold whole request bodies, and an event list never shows them. fn load_filtered_events( connection: &mut PgConnection, filters: &EventFilters<'_>, + include_content: bool, ) -> Result, AppError> { let event_type = filters .event_type @@ -856,12 +1058,13 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(to)); } - if let Some(agent_name) = filters.agent_name.and_then(non_empty) { - let normalized = agent_name_filter_values(agent_name); - query = query.filter(llm_usage_events::agent_name.eq_any(normalized)); + let agent_names = exact_values_filter(filters.agent_names); + if !agent_names.is_empty() { + query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } - if let Some(session_id) = filters.session_id.and_then(non_empty) { - query = query.filter(llm_usage_events::session_id.eq(session_id)); + let session_ids = exact_values_filter(filters.session_ids); + if !session_ids.is_empty() { + query = query.filter(llm_usage_events::session_id.eq_any(session_ids)); } if let Some(session_pk) = filters.session_pk { query = query.filter(llm_usage_events::session_pk.eq(session_pk)); @@ -869,21 +1072,55 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - if let Some(provider) = filters.llm_provider.and_then(non_empty) { - let normalized = normalize_slug(provider); - query = query.filter(llm_usage_events::llm_provider.eq(normalized)); + let providers = exact_values_filter(filters.llm_providers); + if !providers.is_empty() { + query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } - if let Some(model) = filters.llm_model.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_model.eq(model.trim())); + let models = exact_values_filter(filters.llm_models); + if !models.is_empty() { + query = query.filter(llm_usage_events::llm_model.eq_any(models)); } if let Some(event_type) = event_type { query = query.filter(llm_usage_events::event_type.eq(event_type)); } - query + let query = query .order(llm_usage_events::observed_at.desc()) .limit(filters.limit) - .offset(filters.offset) + .offset(filters.offset); + if include_content { + return query.load::(connection).map_err(AppError::from); + } + query + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::device_context_id, + llm_usage_events::session_pk, + llm_usage_events::turn_pk, + llm_usage_events::event_id, + llm_usage_events::agent_name, + llm_usage_events::agent_version, + llm_usage_events::session_id, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + None::.into_sql::>(), + None::>.into_sql::>(), + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + serde_json::json!({}).into_sql::(), + llm_usage_events::created_at, + llm_usage_events::stream_id, + llm_usage_events::response_id, + llm_usage_events::previous_response_id, + )) .load::(connection) .map_err(AppError::from) } @@ -965,7 +1202,7 @@ fn event_response( platform: device.map(|value| value.platform.clone()), session_pk: event.session_pk, turn_pk: event.turn_pk, - agent_name: canonical_agent_name(&event.agent_name), + agent_name: event.agent_name, agent_version: event.agent_version, session_id: event.session_id, turn_index: event.turn_index, @@ -981,6 +1218,9 @@ fn event_response( reasoning_tokens: event.reasoning_tokens, total_tokens: event.total_tokens, observed_at: event.observed_at, + stream_id: event.stream_id, + response_id: event.response_id, + previous_response_id: event.previous_response_id, metadata: event.metadata, attachments, } @@ -1029,12 +1269,13 @@ struct EventFilters<'a> { from: Option>, to: Option>, user_filter: Option<&'a str>, - agent_name: Option<&'a str>, - session_id: Option<&'a str>, + /// Each list matches any of its values; an empty list matches everything. + agent_names: &'a [String], + session_ids: &'a [String], session_pk: Option, turn_index: Option, - llm_provider: Option<&'a str>, - llm_model: Option<&'a str>, + llm_providers: &'a [String], + llm_models: &'a [String], event_type: Option<&'a str>, limit: i64, offset: i64, @@ -1095,11 +1336,11 @@ struct SummarySqlFilters { user_filter_id: Option, user_filter_pattern: Option, agent_names: Vec, - session_id: Option, + session_ids: Vec, session_pk: Option, turn_index: Option, - provider: Option, - model: Option, + providers: Vec, + models: Vec, event_type: Option, } @@ -1109,20 +1350,10 @@ impl SummarySqlFilters { let user_filter_id = user_filter.and_then(|value| Uuid::parse_str(value).ok()); let user_filter_pattern = user_filter.map(|value| format!("%{}%", escape_like_pattern(value))); - let agent_names = filters - .agent_name - .and_then(non_empty) - .map(agent_name_filter_values) - .unwrap_or_default(); - let session_id = filters - .session_id - .and_then(non_empty) - .map(|value| value.trim().to_owned()); - let provider = filters.llm_provider.and_then(non_empty).map(normalize_slug); - let model = filters - .llm_model - .and_then(non_empty) - .map(|value| value.trim().to_owned()); + let agent_names = exact_values_filter(filters.agent_names); + let session_ids = exact_values_filter(filters.session_ids); + let providers = exact_values_filter(filters.llm_providers); + let models = exact_values_filter(filters.llm_models); let event_type = filters.event_type.map(validate_event_type).transpose()?; Ok(Self { @@ -1132,11 +1363,11 @@ impl SummarySqlFilters { user_filter_id, user_filter_pattern, agent_names, - session_id, + session_ids, session_pk: filters.session_pk, turn_index: filters.turn_index, - provider, - model, + providers, + models, event_type, }) } @@ -1216,12 +1447,6 @@ impl From for SummaryRow { } } -#[derive(QueryableByName)] -struct TurnIndexRow { - #[diesel(sql_type = Integer)] - turn_index: i32, -} - struct UpsertSessionInput<'a> { device: &'a Device, user_id: Uuid, @@ -1237,10 +1462,8 @@ mod tests { use std::collections::{HashMap, HashSet}; use super::{ - EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, TurnIdentityKind, - agent_name_filter_values, canonical_agent_name, choose_turn_index, event_response, - latest_optional, normalize_limit, normalize_slug, normalized_tokens, parse_group_by, - turn_identity_from_metadata, validate_batch, validate_event, validate_event_type, + EventFilters, SummaryDimension, SummaryDimensions, SummarySqlFilters, event_response, + latest_optional, normalize_limit, parse_group_by, validate_batch, validate_event_type, }; use chrono::{DateTime, TimeZone, Utc}; use serde_json::json; @@ -1396,43 +1619,13 @@ mod tests { os_version: includes("device") .then(|| device.and_then(|value| value.os_version.clone())) .flatten(), - agent_name: includes("agent").then(|| canonical_agent_name(&event.agent_name)), + agent_name: includes("agent").then(|| event.agent_name.clone()), llm_provider: includes("provider").then(|| event.llm_provider.clone()), llm_model: includes("model").then(|| event.llm_model.clone()), event_type: includes("event_type").then(|| event.event_type.clone()), } } - #[test] - fn normalize_slug_converts_agent_names() { - assert_eq!( - normalize_slug("Claude Code"), - "claude-code", - "agent names should be normalized to lowercase kebab-case" - ); - } - - #[test] - fn canonical_agent_name_groups_claude_clients() { - assert_eq!(canonical_agent_name("Claude"), "claude-code"); - assert_eq!(canonical_agent_name("Claude Code"), "claude-code"); - assert_eq!(canonical_agent_name("claude-desktop"), "claude-code"); - assert_eq!(canonical_agent_name("Codex"), "codex"); - } - - #[test] - fn agent_name_filter_values_include_historical_claude_slugs() { - assert_eq!( - agent_name_filter_values("Claude Code"), - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] - ); - assert_eq!(agent_name_filter_values("Codex"), vec!["codex".to_owned()]); - } - #[test] fn latest_optional_keeps_the_newer_timestamp() { let early = Utc @@ -1484,18 +1677,22 @@ mod tests { } #[test] - fn summary_sql_filters_normalize_runtime_filters() { + fn summary_sql_filters_trim_and_dedupe_runtime_filters() { let filters = EventFilters { user_id: fixed_uuid(42_u128), from: Some(timestamp(2_u32, 0_u32, 0_u32)), to: None, user_filter: Some(" Alice_% "), - agent_name: Some("Claude Desktop"), - session_id: Some(" session-1 "), + agent_names: &[" claude-desktop ".to_owned(), "codex".to_owned()], + session_ids: &[" session-1 ".to_owned(), "session-2".to_owned()], session_pk: None, turn_index: None, - llm_provider: Some("OpenAI"), - llm_model: Some(" gpt-5.5 "), + llm_providers: &["OpenAI".to_owned(), " ".to_owned()], + llm_models: &[ + " gpt-5.5 ".to_owned(), + "gpt-5.5".to_owned(), + "o3".to_owned(), + ], event_type: Some(" Response "), limit: 50_i64, offset: 0_i64, @@ -1512,34 +1709,15 @@ mod tests { sql_filters.user_filter_pattern.as_deref(), Some("%Alice\\_\\%%") ); - assert_eq!( - sql_filters.agent_names, - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] - ); - assert_eq!(sql_filters.session_id.as_deref(), Some("session-1")); + assert_eq!(sql_filters.agent_names, ["claude-desktop", "codex"]); + assert_eq!(sql_filters.session_ids, ["session-1", "session-2"]); assert_eq!(sql_filters.session_pk, None); assert_eq!(sql_filters.turn_index, None); - assert_eq!(sql_filters.provider.as_deref(), Some("openai")); - assert_eq!(sql_filters.model.as_deref(), Some("gpt-5.5")); + assert_eq!(sql_filters.providers, ["OpenAI"]); + assert_eq!(sql_filters.models, ["gpt-5.5", "o3"]); assert_eq!(sql_filters.event_type.as_deref(), Some("response")); } - #[test] - fn validate_event_rejects_zero_turn_index() { - let mut event = sample_event(); - event.turn_index = 0_i32; - - let error = validate_event(&event).expect_err("zero turn_index should be invalid"); - assert!( - matches!(&error, AppError::Validation(message) if message.contains("turn_index")), - "zero turn_index should return a validation error" - ); - } - #[test] fn validate_batch_rejects_oversized_batches() { let request = IngestEventsRequest { @@ -1555,42 +1733,6 @@ mod tests { ); } - #[test] - fn normalized_tokens_derives_total_from_components() { - let mut event = sample_event(); - event.token_usage = TokenUsagePayload { - input_tokens: Some(11_i64), - output_tokens: Some(13_i64), - cache_read_tokens: Some(2_i64), - cache_write_tokens: Some(3_i64), - reasoning_tokens: Some(5_i64), - total_tokens: None, - }; - - let tokens = normalized_tokens(&event).expect("token usage should normalize"); - assert_eq!( - tokens.total, 34_i64, - "missing total_tokens should be derived from all token components" - ); - } - - #[test] - fn normalized_tokens_rejects_negative_components() { - let mut event = sample_event(); - event.token_usage = TokenUsagePayload { - input_tokens: Some(-1_i64), - ..TokenUsagePayload::default() - }; - - let Err(error) = normalized_tokens(&event) else { - panic!("negative token usage should be invalid"); - }; - assert!( - matches!(&error, AppError::Validation(message) if message.contains("input_tokens")), - "negative input_tokens should return a validation error" - ); - } - #[test] fn validate_event_type_normalizes_known_values() { assert_eq!( @@ -1615,74 +1757,6 @@ mod tests { ); } - #[test] - fn choose_turn_index_reuses_existing_provider_identity() { - assert_eq!( - choose_turn_index(9_i32, Some(1_i32), 5_i32), - 1_i32, - "request and response events for the same provider response should share one turn" - ); - } - - #[test] - fn choose_turn_index_allocates_after_conflicting_restarted_counter() { - assert_eq!( - choose_turn_index(1_i32, None, 5_i32), - 5_i32, - "a restarted collector must not merge a new provider response into an old turn" - ); - } - - #[test] - fn choose_turn_index_allocates_after_restarted_counter_gap() { - assert_eq!( - choose_turn_index(2_i32, None, 5_i32), - 5_i32, - "a restarted collector should append even if earlier filtered turns left a gap" - ); - } - - #[test] - fn choose_turn_index_keeps_requested_gap_when_there_is_no_collision() { - assert_eq!( - choose_turn_index(4_i32, None, 2_i32), - 4_i32, - "non-conflicting collector indexes should remain stable" - ); - } - - #[test] - fn turn_identity_prefers_codex_turn_id_before_provider_response_id() { - let identity = turn_identity_from_metadata(&json!({ - "codex_turn_id": "turn-a", - "response_id": "resp-a", - "request_hash": "hash-a" - })) - .expect("codex_turn_id should produce a turn identity"); - - assert!( - matches!(identity.kind, TurnIdentityKind::CodexTurnId), - "one Codex turn can span multiple provider response ids" - ); - assert_eq!(identity.value, "turn-a"); - } - - #[test] - fn turn_identity_uses_claude_turn_id_before_provider_message_id() { - let identity = turn_identity_from_metadata(&json!({ - "claude_turn_id": "claude-turn-a", - "message_id": "msg-a", - "request_hash": "hash-a" - })) - .expect("claude_turn_id should produce a turn identity"); - - assert!( - matches!(identity.kind, TurnIdentityKind::ClaudeTurnId), - "one Claude user turn can span multiple provider message ids" - ); - assert_eq!(identity.value, "claude-turn-a"); - } - #[test] fn normalize_limit_uses_fallback_and_clamps_bounds() { assert_eq!( @@ -1773,7 +1847,7 @@ mod tests { } #[test] - fn aggregate_summary_groups_historical_claude_agent_slugs() { + fn aggregate_summary_keeps_claude_desktop_apart_from_claude_code() { let device_context_id = fixed_uuid(1_u128); let mut desktop_event = usage_event( "evt-claude-desktop-request", @@ -1807,16 +1881,25 @@ mod tests { &group_by, ); + let rows = rows + .iter() + .map(|row| { + ( + row.agent_name.as_deref(), + row.requests, + row.responses, + row.total_tokens, + ) + }) + .collect::>(); assert_eq!( - rows.len(), - 1_usize, - "Claude CLI and Desktop rows should aggregate as one product" + rows, + [ + (Some("claude-code"), 0_i64, 1_i64, 12_i64), + (Some("claude-desktop"), 1_i64, 0_i64, 8_i64), + ], + "Claude Desktop is its own Agent, not an alias of Claude Code" ); - let row = rows.first().expect("summary should contain one row"); - assert_eq!(row.agent_name.as_deref(), Some("claude-code")); - assert_eq!(row.requests, 1_i64); - assert_eq!(row.responses, 1_i64); - assert_eq!(row.total_tokens, 20_i64); } #[test] @@ -1867,7 +1950,11 @@ mod tests { version: Some("1.0.0".to_owned()), }, session_id: "session-001".to_owned(), - turn_index: 1_i32, + turn_id: None, + naming_thread: false, + stream_id: None, + response_id: None, + previous_response_id: None, llm: LlmPayload { provider: "OpenAI".to_owned(), model: "gpt-5.5".to_owned(), @@ -1877,10 +1964,10 @@ mod tests { token_usage: TokenUsagePayload { input_tokens: Some(5_i64), output_tokens: Some(7_i64), - cache_read_tokens: None, - cache_write_tokens: None, - reasoning_tokens: None, - total_tokens: None, + cache_read_tokens: Some(0_i64), + cache_write_tokens: Some(0_i64), + reasoning_tokens: Some(0_i64), + total_tokens: Some(12_i64), }, metadata: json!({}), attachments: Vec::new(), @@ -1920,6 +2007,9 @@ mod tests { observed_at: timestamp(1_u32, 0_u32, 0_u32), metadata: json!({}), created_at: timestamp(1_u32, 0_u32, 1_u32), + stream_id: None, + response_id: None, + previous_response_id: None, } } diff --git a/crates/abyss-backend/src/storage/postgres/search/document.rs b/crates/abyss-backend/src/storage/postgres/search/document.rs index 0c0d7ec..1120039 100644 --- a/crates/abyss-backend/src/storage/postgres/search/document.rs +++ b/crates/abyss-backend/src/storage/postgres/search/document.rs @@ -31,9 +31,9 @@ pub struct SearchDocument { pub turn_pk: Uuid, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical Agent name used for exact filtering. + /// Agent name as reported by the collector, used for exact filtering. pub agent_name: String, - /// Canonical LLM provider slug used for exact filtering. + /// LLM provider as reported by the collector, used for exact filtering. pub llm_provider: String, /// Provider-specific model identifier used for exact filtering. pub llm_model: String, @@ -50,12 +50,6 @@ pub struct SearchDocument { /// Bounded raw tool inputs and outputs. #[serde(skip_serializing_if = "Vec::is_empty")] pub tool_content: Vec, - /// Bounded command strings extracted from structured tool input. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub commands: Vec, - /// Bounded paths and working directories extracted from allowlisted fields. - #[serde(skip_serializing_if = "Vec::is_empty")] - pub file_paths: Vec, } impl SearchDocument { @@ -78,8 +72,6 @@ impl SearchDocument { content: projection.content, tool_names: projection.tool_names, tool_content: projection.tool_content, - commands: projection.commands, - file_paths: projection.file_paths, } } } @@ -98,7 +90,6 @@ mod tests { fn extracts_only_searchable_tool_metadata() { let mut event = usage_event(); event.metadata = json!({ - "working_directory": "/Users/alice/repo", "authorization": "Bearer secret", "content_segments": [ { @@ -120,8 +111,11 @@ mod tests { let document = SearchDocument::from_event(event); assert_eq!(document.tool_names, ["Bash"]); - assert_eq!(document.commands, ["cargo test"]); - assert_eq!(document.file_paths, ["/Users/alice/repo", "src/lib.rs"]); + // Tool inputs are searchable as written; no field is pulled out by name. + assert_eq!( + document.tool_content[0], + "{\"command\":\"cargo test\",\"nested\":{\"file_path\":\"src/lib.rs\"}}" + ); assert_eq!(document.tool_content.len(), 2); let serialized = serde_json::to_string(&document).expect("document should serialize"); assert!(!serialized.contains("Bearer secret")); @@ -153,7 +147,6 @@ mod tests { "before {HIGHLIGHT_START}match{HIGHLIGHT_END} after" )); event.metadata = json!({ - "working_directory": format!("/tmp/{HIGHLIGHT_START}repo"), "content_segments": [ { "type": "tool_call", @@ -204,6 +197,9 @@ mod tests { observed_at: Utc::now(), metadata: json!({}), created_at: Utc::now(), + stream_id: None, + response_id: None, + previous_response_id: None, } } } diff --git a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs index ee59dfa..8757c4b 100644 --- a/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs +++ b/crates/abyss-backend/src/storage/postgres/search/elasticsearch.rs @@ -246,7 +246,7 @@ pub struct MatchedEvent { pub turn_index: i32, /// Request or response side. pub event_type: String, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -300,9 +300,7 @@ fn index_definition() -> Value { "observed_at": {"type": "date"}, "content": {"type": "text"}, "tool_names": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, - "tool_content": {"type": "text"}, - "commands": {"type": "text"}, - "file_paths": {"type": "text"} + "tool_content": {"type": "text"} } } }) @@ -322,9 +320,9 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { } filters.push(json!({"range": {"observed_at": Value::Object(range)}})); } - push_term_filter(&mut filters, "agent_name", query.agent_name.as_deref()); - push_term_filter(&mut filters, "llm_provider", query.llm_provider.as_deref()); - push_term_filter(&mut filters, "llm_model", query.llm_model.as_deref()); + push_terms_filter(&mut filters, "agent_name", &query.agent_names); + push_terms_filter(&mut filters, "llm_provider", &query.llm_providers); + push_terms_filter(&mut filters, "llm_model", &query.llm_models); push_term_filter(&mut filters, "event_type", query.event_type.as_deref()); json!({ @@ -341,8 +339,6 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { "fields": [ "content^4", "session_id^3", - "commands^3", - "file_paths^3", "tool_names^2", "tool_content" ], @@ -375,8 +371,6 @@ fn search_request(user_id: Uuid, query: &ValidatedSearchQuery) -> Value { "number_of_fragments": 2, "fields": { "content": {}, - "commands": {}, - "file_paths": {}, "tool_names": {}, "tool_content": {} } @@ -402,6 +396,15 @@ fn push_term_filter(filters: &mut Vec, field: &str, value: Option<&str>) } } +/// Matches any of `values`; an empty list adds no filter. +fn push_terms_filter(filters: &mut Vec, field: &str, values: &[String]) { + if !values.is_empty() { + let mut terms = serde_json::Map::new(); + terms.insert(field.to_owned(), json!(values)); + filters.push(json!({"terms": Value::Object(terms)})); + } +} + fn append_json_line( body: &mut String, value: &T, @@ -594,17 +597,11 @@ impl TryFrom for SearchMatchPage { } fn ordered_fragments(highlights: &HashMap>) -> Vec { - [ - "content", - "commands", - "file_paths", - "tool_names", - "tool_content", - ] - .into_iter() - .flat_map(|field| highlights.get(field).into_iter().flatten().cloned()) - .take(2) - .collect() + ["content", "tool_names", "tool_content"] + .into_iter() + .flat_map(|field| highlights.get(field).into_iter().flatten().cloned()) + .take(2) + .collect() } #[cfg(test)] @@ -625,9 +622,9 @@ mod tests { text: "timeout".to_owned(), from: Some(Utc.with_ymd_and_hms(2026, 8, 1, 0, 0, 0).unwrap()), to: None, - agent_name: Some("claude-code".to_owned()), - llm_provider: None, - llm_model: None, + agent_names: vec!["claude-code".to_owned(), "codex".to_owned()], + llm_providers: Vec::new(), + llm_models: vec!["gpt-5".to_owned()], event_type: None, page: 1, page_size: 20, @@ -640,7 +637,13 @@ mod tests { .as_array() .expect("filters should be an array"); assert!(filters.contains(&json!({"term": {"user_id": user_id}}))); - assert!(filters.contains(&json!({"term": {"agent_name": "claude-code"}}))); + assert!(filters.contains(&json!({"terms": {"agent_name": ["claude-code", "codex"]}}))); + assert!(filters.contains(&json!({"terms": {"llm_model": ["gpt-5"]}}))); + assert!( + !filters + .iter() + .any(|filter| filter["terms"].get("llm_provider").is_some()) + ); assert_eq!(request["collapse"]["field"], "session_pk"); } diff --git a/crates/abyss-backend/src/storage/postgres/search/outbox.rs b/crates/abyss-backend/src/storage/postgres/search/outbox.rs index d7e2876..f9187a7 100644 --- a/crates/abyss-backend/src/storage/postgres/search/outbox.rs +++ b/crates/abyss-backend/src/storage/postgres/search/outbox.rs @@ -10,9 +10,11 @@ use std::collections::HashMap; use chrono::{DateTime, Duration, Utc}; use diesel::{ - ExpressionMethods, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, + ExpressionMethods, IntoSql, PgConnection, QueryDsl, QueryableByName, RunQueryDsl, + SelectableHelper, + dsl::{exists, not}, sql_query, - sql_types::{BigInt, Bool, Integer, Text, Uuid as SqlUuid}, + sql_types::{BigInt, Integer, Text, Uuid as SqlUuid}, }; use uuid::Uuid; @@ -68,42 +70,45 @@ impl SearchOutboxRepository { connection: &mut PgConnection, batch_size: i64, ) -> Result { + let queued_upsert = search_outbox::table + .filter(search_outbox::event_pk.eq(llm_usage_events::id)) + .filter(search_outbox::operation.eq("upsert")) + .select(search_outbox::id); // The unique (event_pk, operation) constraint makes every backfill pass // idempotent and allows startup by multiple replicas. - let queued = sql_query( - "INSERT INTO search_outbox (event_pk, user_id, operation, created_at) \ - SELECT source.id, source.user_id, 'upsert', source.created_at \ - FROM llm_usage_events AS source \ - WHERE NOT EXISTS (\ - SELECT 1 \ - FROM search_outbox AS queued \ - WHERE queued.event_pk = source.id \ - AND queued.operation = 'upsert'\ - ) \ - ORDER BY source.id \ - LIMIT $1 \ - ON CONFLICT (event_pk, operation) DO NOTHING", - ) - .bind::(batch_size) - .execute(connection)?; + let queued = diesel::insert_into(search_outbox::table) + .values( + llm_usage_events::table + .filter(not(exists(queued_upsert))) + .order(llm_usage_events::id) + .limit(batch_size) + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + "upsert".into_sql::(), + llm_usage_events::created_at, + )), + ) + .into_columns(( + search_outbox::event_pk, + search_outbox::user_id, + search_outbox::operation, + search_outbox::created_at, + )) + .on_conflict((search_outbox::event_pk, search_outbox::operation)) + .do_nothing() + .execute(connection)?; if i64::try_from(queued).unwrap_or(i64::MAX) >= batch_size { return Ok(false); } - let state = sql_query( - "SELECT NOT EXISTS (\ - SELECT 1 \ - FROM llm_usage_events AS source \ - WHERE NOT EXISTS (\ - SELECT 1 \ - FROM search_outbox AS queued \ - WHERE queued.event_pk = source.id \ - AND queued.operation = 'upsert'\ - )\ - ) AS complete", - ) - .get_result::(connection)?; - Ok(state.complete) + diesel::select(not(exists( + llm_usage_events::table + .filter(not(exists(queued_upsert))) + .select(llm_usage_events::id), + ))) + .get_result::(connection) + .map_err(AppError::from) } /// Leases pending rows and resolves their source events into bulk operations. @@ -313,12 +318,6 @@ struct RawClaimedOutboxTask { attempt_count: i32, } -#[derive(QueryableByName)] -struct RawBackfillState { - #[diesel(sql_type = Bool)] - complete: bool, -} - fn retry_delay(attempt_count: i32) -> i64 { let exponent = u32::try_from(attempt_count.clamp(1_i32, 8_i32)).unwrap_or(8_u32); 2_i64.saturating_pow(exponent).min(MAX_RETRY_DELAY_SECONDS) diff --git a/crates/abyss-backend/src/storage/postgres/session_list.rs b/crates/abyss-backend/src/storage/postgres/session_list.rs new file mode 100644 index 0000000..112ef4d --- /dev/null +++ b/crates/abyss-backend/src/storage/postgres/session_list.rs @@ -0,0 +1,160 @@ +//! PostgreSQL session index ordered by latest activity. + +use chrono::{DateTime, Utc}; +use diesel::{ + PgConnection, QueryableByName, RunQueryDsl, sql_query, + sql_types::{Array, BigInt, Bool, Nullable, Text, Timestamptz, Uuid as SqlUuid}, +}; +use uuid::Uuid; + +use crate::{ + error::AppError, + usage::session_list::{SessionListItem, SessionListResponse, ValidatedSessionListQuery}, +}; + +/// Per-session activity with the filters that apply before paging. +/// +/// Bind order: `$1` owner, `$2`/`$3` agent filter flag and names, `$4`/`$5` +/// lower activity bound, `$6`/`$7` upper start bound, `$8`/`$9` providers and +/// `$10`/`$11` models a session must have used at least one of. +const ACTIVITY_CTE: &str = "\ + WITH activity AS ( \ + SELECT \ + s.id AS session_pk, \ + s.session_id AS session_id, \ + s.agent_name AS agent_name, \ + s.agent_version AS agent_version, \ + d.host_name AS host_name, \ + d.platform AS platform, \ + s.started_at AS started_at, \ + s.ended_at AS ended_at, \ + COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ + (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id)::bigint AS turns, \ + COUNT(e.id)::bigint AS events, \ + COALESCE(SUM(e.total_tokens), 0)::bigint AS total_tokens, \ + st.title AS title, \ + st.purpose AS purpose \ + FROM agent_sessions s \ + JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN session_titles st ON st.session_pk = s.id \ + LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ + WHERE s.user_id = $1 \ + AND ($2 = false OR s.agent_name = ANY($3)) \ + AND ($8 = false OR EXISTS ( \ + SELECT 1 FROM llm_usage_events p \ + WHERE p.session_pk = s.id AND p.llm_provider = ANY($9))) \ + AND ($10 = false OR EXISTS ( \ + SELECT 1 FROM llm_usage_events m \ + WHERE m.session_pk = s.id AND m.llm_model = ANY($11))) \ + GROUP BY s.id, d.host_name, d.platform, st.title, st.purpose \ + ), \ + filtered AS ( \ + SELECT * FROM activity \ + WHERE ($4 = false OR last_activity_at >= $5) \ + AND ($6 = false OR started_at < $7) \ + ) "; + +#[derive(QueryableByName)] +struct SessionListRow { + #[diesel(sql_type = SqlUuid)] + session_pk: Uuid, + #[diesel(sql_type = Text)] + session_id: String, + #[diesel(sql_type = Text)] + agent_name: String, + #[diesel(sql_type = Nullable)] + agent_version: Option, + #[diesel(sql_type = Text)] + host_name: String, + #[diesel(sql_type = Text)] + platform: String, + #[diesel(sql_type = Timestamptz)] + started_at: DateTime, + #[diesel(sql_type = Nullable)] + ended_at: Option>, + #[diesel(sql_type = Timestamptz)] + last_activity_at: DateTime, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + events: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, + #[diesel(sql_type = Nullable)] + title: Option, + #[diesel(sql_type = Nullable)] + purpose: Option, +} + +#[derive(QueryableByName)] +struct CountRow { + #[diesel(sql_type = BigInt)] + total: i64, +} + +pub(super) fn list_sessions( + connection: &mut PgConnection, + user_id: Uuid, + query: &ValidatedSessionListQuery, +) -> Result { + let has_agents = !query.agent_names.is_empty(); + let from = query.from.unwrap_or(DateTime::UNIX_EPOCH); + let to = query.to.unwrap_or(DateTime::UNIX_EPOCH); + + let total = sql_query(format!( + "{ACTIVITY_CTE} SELECT COUNT(*)::bigint AS total FROM filtered" + )) + .bind::(user_id) + .bind::(has_agents) + .bind::, _>(&query.agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(!query.llm_providers.is_empty()) + .bind::, _>(&query.llm_providers) + .bind::(!query.llm_models.is_empty()) + .bind::, _>(&query.llm_models) + .get_result::(connection)? + .total; + + let rows = sql_query(format!( + "{ACTIVITY_CTE} SELECT * FROM filtered \ + ORDER BY last_activity_at DESC, session_pk DESC LIMIT $12 OFFSET $13" + )) + .bind::(user_id) + .bind::(has_agents) + .bind::, _>(&query.agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(!query.llm_providers.is_empty()) + .bind::, _>(&query.llm_providers) + .bind::(!query.llm_models.is_empty()) + .bind::, _>(&query.llm_models) + .bind::(i64::from(query.page_size)) + .bind::(query.offset) + .load::(connection)?; + + let items = rows + .into_iter() + .map(|row| SessionListItem { + session_pk: row.session_pk, + session_id: row.session_id, + agent_name: row.agent_name, + agent_version: row.agent_version, + host_name: row.host_name, + platform: row.platform, + started_at: row.started_at, + ended_at: row.ended_at, + last_activity_at: row.last_activity_at, + turns: row.turns, + events: row.events, + total_tokens: row.total_tokens, + title: row.title, + purpose: row.purpose, + }) + .collect(); + Ok(SessionListResponse::new(query, total, items)) +} diff --git a/crates/abyss-backend/src/storage/postgres/session_titles.rs b/crates/abyss-backend/src/storage/postgres/session_titles.rs new file mode 100644 index 0000000..f9686f0 --- /dev/null +++ b/crates/abyss-backend/src/storage/postgres/session_titles.rs @@ -0,0 +1,157 @@ +//! PostgreSQL persistence of session names reported by the collector. + +use chrono::{DateTime, Utc}; +use diesel::{ExpressionMethods, OptionalExtension, PgConnection, QueryDsl, RunQueryDsl}; +use serde_json::Value; +use uuid::Uuid; + +use crate::{ + db::schema::{llm_usage_events, session_titles}, + error::AppError, + usage::{ + SessionInfo, + session_title::{TITLE_GENERATION_PURPOSE, TitleContribution, default_title}, + }, +}; + +/// Stored title, purpose, and matched task session of one session. +type StoredTitle = (Option, Option, Option); + +/// One stored event considered for title capture. +pub(super) struct CapturedEvent<'a> { + pub(super) user_id: Uuid, + pub(super) session_pk: Uuid, + pub(super) event_pk: Uuid, + pub(super) event_type: &'a str, + pub(super) metadata: &'a Value, + pub(super) naming_thread: bool, + pub(super) observed_at: DateTime, +} + +/// Records what a newly stored event says about its session's name. +/// +/// An Agent title replaces a default title and an older Agent title. A default +/// title keeps the earliest user input, even after late delivery, and never +/// replaces an Agent title. +pub(super) fn capture( + connection: &mut PgConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + let session = event.session_pk; + let agent_title = + match TitleContribution::of(event.event_type, event.metadata, event.naming_thread) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; + let stored = session_titles::table + .find(session) + .select(( + session_titles::title_event_pk, + session_titles::title_observed_at, + )) + .first::<(Option, Option>)>(connection) + .optional()? + .and_then(|(event_pk, observed_at)| event_pk.zip(observed_at)); + let agent = agent_title.is_some(); + let replaces = match stored { + None => true, + Some((_, stored_at)) if agent && stored_at < event.observed_at => true, + Some((_, stored_at)) if !agent && stored_at <= event.observed_at => false, + // Otherwise only a stored default title gives way. + Some((event_pk, _)) => !is_agent_title(connection, event_pk)?, + }; + if !replaces { + return Ok(()); + } + // Read the user input only once it could name the session. + let Some(title) = agent_title + .map(str::to_owned) + .or_else(|| default_title(event.metadata)) + else { + return Ok(()); + }; + let now = Utc::now(); + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(event.user_id), + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + Ok(()) +} + +/// Agent titles are read from responses and default titles from requests, so +/// the side of the event that gave the stored title tells them apart. +fn is_agent_title(connection: &mut PgConnection, event_pk: Uuid) -> Result { + Ok(llm_usage_events::table + .find(event_pk) + .select(llm_usage_events::event_type) + .first::(connection) + .optional()? + .is_some_and(|event_type| event_type == "response")) +} + +/// Marks a session as a thread that names another session. +fn mark_title_thread( + connection: &mut PgConnection, + session: Uuid, + user_id: Uuid, +) -> Result<(), AppError> { + let now = Utc::now(); + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(user_id), + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + Ok(()) +} + +/// Adds the stored title, purpose, and matched task session to a session. +pub(super) fn fill( + connection: &mut PgConnection, + session: &mut SessionInfo, +) -> Result<(), AppError> { + (session.title, session.purpose, session.parent_session_pk) = + load(connection, session.session_pk)?; + Ok(()) +} + +/// Returns the stored title and purpose of one session, and for a naming +/// thread the task session it was matched to. +fn load(connection: &mut PgConnection, session_pk: Uuid) -> Result { + Ok(session_titles::table + .find(session_pk) + .select(( + session_titles::title, + session_titles::purpose, + session_titles::parent_session_pk, + )) + .first::<(Option, Option, Option)>(connection) + .optional()? + .unwrap_or((None, None, None))) +} diff --git a/crates/abyss-backend/src/storage/sqlite/mod.rs b/crates/abyss-backend/src/storage/sqlite/mod.rs index 4214d9c..11f2f31 100644 --- a/crates/abyss-backend/src/storage/sqlite/mod.rs +++ b/crates/abyss-backend/src/storage/sqlite/mod.rs @@ -6,6 +6,8 @@ mod models; mod repository; mod schema; mod search; +mod session_list; +mod session_titles; use diesel::{QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star}; use uuid::Uuid; @@ -17,7 +19,10 @@ use crate::{ storage::{BoxedFuture, StorageBackend}, usage::{ IngestEventsRequest, IngestEventsResponse, RawEventsQuery, RawEventsResponse, - SessionTimelineResponse, SummaryQuery, SummaryResponse, attachments::StoredImageAttachment, + SessionOutlineResponse, SessionTimelineResponse, SummaryQuery, SummaryResponse, + UsageEventResponse, + attachments::StoredImageAttachment, + session_list::{SessionListResponse, ValidatedSessionListQuery}, }, }; @@ -97,6 +102,17 @@ impl StorageBackend for SqliteFtsBackend { }) } + fn usage_event( + &self, + user_id: Uuid, + event_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| repository::usage_event(connection, user_id, event_pk)) + .await + }) + } + fn usage_summary( &self, user_id: Uuid, @@ -124,6 +140,30 @@ impl StorageBackend for SqliteFtsBackend { }) } + fn session_outline( + &self, + user_id: Uuid, + session_pk: Uuid, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| { + repository::session_outline(connection, user_id, session_pk) + }) + .await + }) + } + + fn list_sessions( + &self, + user_id: Uuid, + query: ValidatedSessionListQuery, + ) -> BoxedFuture<'_, Result> { + Box::pin(async move { + self.run_db(move |connection| session_list::list_sessions(connection, user_id, &query)) + .await + }) + } + fn image_attachment( &self, user_id: Uuid, diff --git a/crates/abyss-backend/src/storage/sqlite/models.rs b/crates/abyss-backend/src/storage/sqlite/models.rs index ca70f21..0bc692f 100644 --- a/crates/abyss-backend/src/storage/sqlite/models.rs +++ b/crates/abyss-backend/src/storage/sqlite/models.rs @@ -82,6 +82,7 @@ pub(super) struct TurnRow { pub(super) user_id: String, pub(super) session_pk: String, pub(super) turn_index: i32, + pub(super) turn_key: Option, pub(super) started_at: i64, pub(super) ended_at: Option, pub(super) created_at: i64, @@ -95,6 +96,7 @@ pub(super) struct NewTurn { pub(super) user_id: String, pub(super) session_pk: String, pub(super) turn_index: i32, + pub(super) turn_key: Option, pub(super) started_at: i64, pub(super) ended_at: Option, pub(super) created_at: i64, @@ -129,6 +131,9 @@ pub(super) struct EventRow { pub(super) observed_at: i64, pub(super) metadata: String, pub(super) created_at: i64, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } #[derive(Insertable)] @@ -158,6 +163,9 @@ pub(super) struct NewEvent { pub(super) observed_at: i64, pub(super) metadata: String, pub(super) created_at: i64, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } #[derive(Queryable, Selectable)] @@ -215,7 +223,6 @@ pub(super) struct DeviceRecord { pub(super) id: Uuid, pub(super) host_name: String, pub(super) platform: String, - pub(super) os_version: Option, } pub(super) struct SessionRecord { @@ -262,6 +269,9 @@ pub(super) struct EventRecord { pub(super) total_tokens: i64, pub(super) observed_at: DateTime, pub(super) metadata: Value, + pub(super) stream_id: Option, + pub(super) response_id: Option, + pub(super) previous_response_id: Option, } impl TimelineEvent for EventRecord { @@ -269,10 +279,6 @@ impl TimelineEvent for EventRecord { self.turn_index } - fn metadata(&self) -> &Value { - &self.metadata - } - fn observed_at(&self) -> DateTime { self.observed_at } @@ -298,12 +304,16 @@ pub(super) struct AttachmentRecord { impl DeviceRow { pub(super) fn into_record(self) -> Result { - let _ = (&self.user_id, self.created_at, self.updated_at); + let _ = ( + &self.user_id, + &self.os_version, + self.created_at, + self.updated_at, + ); Ok(DeviceRecord { id: parse_uuid(&self.id, "device id")?, host_name: self.host_name, platform: self.platform, - os_version: self.os_version, }) } } @@ -369,6 +379,9 @@ impl EventRow { total_tokens: self.total_tokens, observed_at: timestamp_from_micros(self.observed_at)?, metadata: parse_json(&self.metadata, "event metadata")?, + stream_id: self.stream_id, + response_id: self.response_id, + previous_response_id: self.previous_response_id, }) } } diff --git a/crates/abyss-backend/src/storage/sqlite/repository.rs b/crates/abyss-backend/src/storage/sqlite/repository.rs index 4c24f29..33690c4 100644 --- a/crates/abyss-backend/src/storage/sqlite/repository.rs +++ b/crates/abyss-backend/src/storage/sqlite/repository.rs @@ -4,8 +4,12 @@ use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; use diesel::{ - ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper, SqliteConnection, dsl::max, - result::OptionalExtension, sql_query, sql_types::Text, + ExpressionMethods, IntoSql, QueryDsl, QueryableByName, RunQueryDsl, SelectableHelper, + SqliteConnection, + dsl::{count_star, max, sql}, + result::OptionalExtension, + sql_query, + sql_types::{BigInt, Binary, Bool, Nullable, Text}, }; use uuid::Uuid; @@ -13,9 +17,10 @@ use crate::{ error::AppError, search::projection::SearchProjection, usage::{ - IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, RawEventsQuery, - RawEventsResponse, SessionInfo, SessionTimelineResponse, SummaryQuery, SummaryResponse, - SummaryRow, TurnTimeline, UsageEventResponse, + IngestEventsRequest, IngestEventsResponse, IngestUsageEvent, OUTLINE_PREVIEW_CHARS, + RawEventsQuery, RawEventsResponse, SessionInfo, SessionOutlineResponse, + SessionTimelineResponse, SummaryQuery, SummaryResponse, SummaryRow, TurnTimeline, + UsageEventOutline, UsageEventResponse, attachments::{ ImageMediaType, StoredImageAttachment, UsageEventAttachmentResponse, ValidatedImageAttachment, @@ -23,25 +28,23 @@ use crate::{ diagnostics::IngestDiagnosticCapture, event_order::UsageEventTimelineOrder, persistence::{ - TurnIdentity, agent_name_filter_values, canonical_agent_name, choose_turn_index, - non_empty, normalize_limit, normalize_slug, normalize_text, normalized_optional, - normalized_tokens, parse_group_by, sha256_bytes, turn_identity, validate_batch, - validate_event_type, + exact_values_filter, non_empty, normalize_limit, normalize_text, parse_group_by, + sha256_bytes, validate_batch, validate_event_type, }, - session_metadata_from_event, }, }; use super::{ models::{ AttachmentRow, DeviceRecord, DeviceRow, EventRecord, EventRow, NewAttachment, NewDevice, - NewDiagnosticCapture, NewDiagnosticCaptureEvent, NewEvent, NewSession, NewTurn, SessionRow, - TurnRow, parse_uuid, + NewDiagnosticCapture, NewDiagnosticCaptureEvent, NewEvent, NewSession, NewTurn, + SessionRecord, SessionRow, TurnRecord, TurnRow, parse_uuid, timestamp_from_micros, }, schema::{ agent_diagnostic_capture_events, agent_diagnostic_captures, agent_sessions, agent_turns, app_users, devices, llm_usage_event_attachments, llm_usage_events, }, + session_titles, }; pub(super) fn ingest_events( @@ -92,7 +95,7 @@ fn ingest_one_event( event: &IngestUsageEvent, attachments: Vec, ) -> Result { - let event_id = normalize_text(&event.event_id); + let event_id = event.event_id.clone(); let exists = llm_usage_events::table .filter(llm_usage_events::event_id.eq(&event_id)) .select(llm_usage_events::id) @@ -105,10 +108,10 @@ fn ingest_one_event( let observed_at = timestamp_to_micros(event.observed_at); let now = timestamp_to_micros(Utc::now()); - let agent_name = canonical_agent_name(&event.agent.name); - let agent_version = normalized_optional(event.agent.version.as_deref()); - let provider = normalize_slug(&event.llm.provider); - let model = normalize_text(&event.llm.model); + let agent_name = event.agent.name.clone(); + let agent_version = event.agent.version.clone(); + let provider = event.llm.provider.clone(); + let model = event.llm.model.clone(); let device_id = upsert_device(connection, user_id, event, observed_at, now)?; let session_pk = upsert_session( connection, @@ -120,17 +123,16 @@ fn ingest_one_event( observed_at, now, )?; - let turn_index = resolve_turn_index(connection, session_pk, event)?; - let turn_pk = upsert_turn( + let (turn_pk, turn_index) = place_in_turn( connection, user_id, session_pk, - turn_index, + event.turn_id.as_deref(), observed_at, now, )?; - let tokens = normalized_tokens(event)?; - let text = normalized_optional(event.text.as_deref()); + let tokens = &event.token_usage; + let text = event.text.clone(); let event_pk = Uuid::now_v7(); let metadata = serialize_json(&event.metadata, "event metadata")?; @@ -142,24 +144,27 @@ fn ingest_one_event( session_pk: session_pk.to_string(), turn_pk: turn_pk.to_string(), event_id, - agent_name, + agent_name: agent_name.clone(), agent_version, - session_id: normalize_text(&event.session_id), + session_id: event.session_id.clone(), turn_index, llm_provider: provider, llm_model: model, event_type: event.event_type.as_str().to_owned(), text: text.clone(), text_sha256: text.as_deref().map(sha256_bytes), - input_tokens: tokens.input, - output_tokens: tokens.output, - cache_read_tokens: tokens.cache_read, - cache_write_tokens: tokens.cache_write, - reasoning_tokens: tokens.reasoning, - total_tokens: tokens.total, + input_tokens: tokens.input_tokens.unwrap_or_default(), + output_tokens: tokens.output_tokens.unwrap_or_default(), + cache_read_tokens: tokens.cache_read_tokens.unwrap_or_default(), + cache_write_tokens: tokens.cache_write_tokens.unwrap_or_default(), + reasoning_tokens: tokens.reasoning_tokens.unwrap_or_default(), + total_tokens: tokens.total_tokens.unwrap_or_default(), observed_at, metadata, created_at: now, + stream_id: event.stream_id.clone(), + response_id: event.response_id.clone(), + previous_response_id: event.previous_response_id.clone(), }) .on_conflict(llm_usage_events::event_id) .do_nothing() @@ -168,6 +173,30 @@ fn ingest_one_event( return Ok(false); } + insert_attachments(connection, user_id, event_pk, attachments, now)?; + insert_search_projection(connection, event_pk, user_id, session_pk, event)?; + session_titles::capture( + connection, + &session_titles::CapturedEvent { + user_id, + session_pk, + event_pk, + event_type: event.event_type.as_str(), + metadata: &event.metadata, + naming_thread: event.naming_thread, + observed_at, + }, + )?; + Ok(true) +} + +fn insert_attachments( + connection: &mut SqliteConnection, + user_id: Uuid, + event_pk: Uuid, + attachments: Vec, + now: i64, +) -> Result<(), AppError> { let attachment_rows = attachments .into_iter() .map(|attachment| NewAttachment { @@ -188,8 +217,7 @@ fn ingest_one_event( .execute(connection)?; } - insert_search_projection(connection, event_pk, user_id, session_pk, event)?; - Ok(true) + Ok(()) } fn upsert_device( @@ -200,9 +228,9 @@ fn upsert_device( now: i64, ) -> Result { let owner = user_id.to_string(); - let host_name = normalize_text(&event.device.host_name); - let platform = normalize_slug(&event.device.platform); - let os_version = normalized_optional(event.device.os_version.as_deref()); + let host_name = event.device.host_name.clone(); + let platform = event.device.platform.clone(); + let os_version = event.device.os_version.clone(); let existing = devices::table .filter(devices::user_id.eq(&owner)) .filter(devices::host_name.eq(&host_name)) @@ -255,7 +283,7 @@ fn upsert_session( now: i64, ) -> Result { let owner = user_id.to_string(); - let session_id = normalize_text(&event.session_id); + let session_id = event.session_id.clone(); let existing = agent_sessions::table .filter(agent_sessions::user_id.eq(&owner)) .filter(agent_sessions::agent_name.eq(agent_name)) @@ -263,10 +291,8 @@ fn upsert_session( .select(SessionRow::as_select()) .first::(connection) .optional()?; - let incoming_metadata = session_metadata_from_event(&event.metadata); if let Some(existing) = existing { let id = parse_uuid(&existing.id, "session id")?; - let metadata = merge_json(&existing.metadata, &incoming_metadata, "session metadata")?; diesel::update(agent_sessions::table.find(existing.id)) .set(( agent_sessions::device_context_id.eq(device_id.to_string()), @@ -276,7 +302,6 @@ fn upsert_session( agent_sessions::ended_at.eq(Some( existing.ended_at.unwrap_or(observed_at).max(observed_at), )), - agent_sessions::metadata.eq(metadata), agent_sessions::updated_at.eq(now), )) .execute(connection)?; @@ -294,7 +319,7 @@ fn upsert_session( session_id, started_at: observed_at, ended_at: Some(observed_at), - metadata: serialize_json(&incoming_metadata, "session metadata")?, + metadata: "{}".to_owned(), created_at: now, updated_at: now, }) @@ -302,95 +327,161 @@ fn upsert_session( Ok(id) } -fn resolve_turn_index( +/// Places an event in a turn of its session and returns the turn's id and +/// position. +/// +/// Events naming the same turn share it, and an event naming none joins the +/// turn in progress when it was observed. Turns are numbered by when they +/// started, so a turn that arrives late moves the turns after it. +fn place_in_turn( connection: &mut SqliteConnection, + user_id: Uuid, session_pk: Uuid, - event: &IngestUsageEvent, -) -> Result { - let Some(identity) = turn_identity(event) else { - return Ok(event.turn_index); + turn_id: Option<&str>, + observed_at: i64, + now: i64, +) -> Result<(Uuid, i32), AppError> { + let session = session_pk.to_string(); + let found = match turn_id { + Some(key) => named_turn(connection, &session, key)?, + None => turn_in_progress(connection, &session, observed_at)?, + }; + let (turn_pk, turn_index, reordered) = if let Some(turn) = found { + diesel::update(agent_turns::table.find(&turn.id)) + .set(( + agent_turns::started_at.eq(turn.started_at.min(observed_at)), + agent_turns::ended_at + .eq(Some(turn.ended_at.unwrap_or(observed_at).max(observed_at))), + agent_turns::updated_at.eq(now), + )) + .execute(connection)?; + (turn.id, turn.turn_index, observed_at < turn.started_at) + } else { + let last = agent_turns::table + .filter(agent_turns::session_pk.eq(&session)) + .select(max(agent_turns::turn_index)) + .first::>(connection)?; + let id = Uuid::now_v7().to_string(); + let turn_index = last.unwrap_or(0_i32).saturating_add(1_i32); + diesel::insert_into(agent_turns::table) + .values(NewTurn { + id: id.clone(), + user_id: user_id.to_string(), + session_pk: session.clone(), + turn_index, + turn_key: turn_id.map(str::to_owned), + started_at: observed_at, + ended_at: Some(observed_at), + created_at: now, + updated_at: now, + }) + .execute(connection)?; + (id, turn_index, last.is_some()) }; - let existing = existing_turn_index_for_identity(connection, session_pk, &identity)?; - let next = agent_turns::table - .filter(agent_turns::session_pk.eq(session_pk.to_string())) - .select(max(agent_turns::turn_index)) - .first::>(connection)? - .unwrap_or(0_i32) - .saturating_add(1); - Ok(choose_turn_index(event.turn_index, existing, next)) + let turn_index = if reordered && number_turns(connection, &session)? { + agent_turns::table + .find(&turn_pk) + .select(agent_turns::turn_index) + .first::(connection)? + } else { + turn_index + }; + Ok((parse_uuid(&turn_pk, "turn id")?, turn_index)) } -fn existing_turn_index_for_identity( +/// The turn named `key` in a session. The first named turn adopts the turn +/// that events naming none opened at the start of the session. +fn named_turn( connection: &mut SqliteConnection, - session_pk: Uuid, - identity: &TurnIdentity, -) -> Result, AppError> { - let rows = llm_usage_events::table - .filter(llm_usage_events::session_pk.eq(session_pk.to_string())) - .order_by(( - llm_usage_events::observed_at.asc(), - llm_usage_events::id.asc(), - )) - .select((llm_usage_events::turn_index, llm_usage_events::metadata)) - .load::<(i32, String)>(connection)?; - for (turn_index, metadata) in rows { - let metadata = serde_json::from_str::(&metadata).map_err(|error| { - AppError::internal(format!("stored event metadata is invalid JSON: {error}")) - })?; - if metadata - .get(identity.kind.metadata_key()) - .and_then(serde_json::Value::as_str) - == Some(identity.value.as_str()) - { - return Ok(Some(turn_index)); - } + session: &str, + key: &str, +) -> Result, AppError> { + let named = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .filter(agent_turns::turn_key.eq(key)) + .select(TurnRow::as_select()) + .first::(connection) + .optional()?; + if named.is_some() { + return Ok(named); } - Ok(None) + let mut turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .select(TurnRow::as_select()) + .limit(2_i64) + .load::(connection)?; + let opening = match turns.as_slice() { + [turn] if turn.turn_key.is_none() => turns.pop(), + _ => None, + }; + if let Some(turn) = &opening { + diesel::update(agent_turns::table.find(&turn.id)) + .set(agent_turns::turn_key.eq(key)) + .execute(connection)?; + } + Ok(opening) } -fn upsert_turn( +/// The turn in progress when an event naming no turn was observed: the latest +/// turn started by then, else the first turn of the session. +fn turn_in_progress( connection: &mut SqliteConnection, - user_id: Uuid, - session_pk: Uuid, - turn_index: i32, + session: &str, observed_at: i64, - now: i64, -) -> Result { - let session = session_pk.to_string(); - let existing = agent_turns::table - .filter(agent_turns::session_pk.eq(&session)) - .filter(agent_turns::turn_index.eq(turn_index)) +) -> Result, AppError> { + let started = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .filter(agent_turns::started_at.le(observed_at)) + .order_by((agent_turns::started_at.desc(), agent_turns::id.desc())) .select(TurnRow::as_select()) .first::(connection) .optional()?; - if let Some(existing) = existing { - let id = parse_uuid(&existing.id, "turn id")?; - diesel::update(agent_turns::table.find(existing.id)) - .set(( - agent_turns::started_at.eq(existing.started_at.min(observed_at)), - agent_turns::ended_at.eq(Some( - existing.ended_at.unwrap_or(observed_at).max(observed_at), - )), - agent_turns::updated_at.eq(now), - )) - .execute(connection)?; - return Ok(id); + if started.is_some() { + return Ok(started); } + agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select(TurnRow::as_select()) + .first::(connection) + .optional() + .map_err(AppError::from) +} - let id = Uuid::now_v7(); - diesel::insert_into(agent_turns::table) - .values(NewTurn { - id: id.to_string(), - user_id: user_id.to_string(), - session_pk: session, - turn_index, - started_at: observed_at, - ended_at: Some(observed_at), - created_at: now, - updated_at: now, - }) - .execute(connection)?; - Ok(id) +/// Numbers the turns of a session, and their events, by when the turns +/// started. Writes nothing and returns `false` while the numbers already +/// follow that order. +fn number_turns(connection: &mut SqliteConnection, session: &str) -> Result { + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session)) + .order_by((agent_turns::started_at.asc(), agent_turns::id.asc())) + .select((agent_turns::id, agent_turns::turn_index)) + .load::<(String, i32)>(connection)?; + let moved = turns + .iter() + .zip(1_i32..) + .filter(|((_, index), position)| index != position) + .collect::>(); + if moved.is_empty() { + return Ok(false); + } + // Moved turns first step above every current number, so no single-row + // update collides with another turn's number or goes below one. + let above = turns.iter().map(|(_, index)| *index).max().unwrap_or(0_i32); + for ((id, _), position) in &moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(above.saturating_add(*position))) + .execute(connection)?; + } + for ((id, _), position) in moved { + diesel::update(agent_turns::table.find(id)) + .set(agent_turns::turn_index.eq(position)) + .execute(connection)?; + diesel::update(llm_usage_events::table.filter(llm_usage_events::turn_pk.eq(id))) + .set(llm_usage_events::turn_index.eq(position)) + .execute(connection)?; + } + Ok(true) } /// Writes the SQLite FTS5 projection. Virtual tables are intentionally outside @@ -402,23 +493,20 @@ fn insert_search_projection( session_pk: Uuid, event: &IngestUsageEvent, ) -> Result<(), AppError> { - let projection = - SearchProjection::from_source(normalized_optional(event.text.as_deref()), &event.metadata); + let projection = SearchProjection::from_source(event.text.clone(), &event.metadata); sql_query( "INSERT INTO usage_events_fts ( event_pk, user_id, session_pk, session_id, content, tool_names, - tool_content, commands, file_paths - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + tool_content + ) VALUES (?, ?, ?, ?, ?, ?, ?)", ) .bind::(event_pk.to_string()) .bind::(user_id.to_string()) .bind::(session_pk.to_string()) - .bind::(normalize_text(&event.session_id)) + .bind::(&event.session_id) .bind::, _>(projection.content) .bind::(projection.tool_names.join("\n")) .bind::(projection.tool_content.join("\n")) - .bind::(projection.commands.join("\n")) - .bind::(projection.file_paths.join("\n")) .execute(connection)?; Ok(()) } @@ -428,14 +516,9 @@ fn ingest_one_diagnostic_capture( user_id: Uuid, capture: &IngestDiagnosticCapture, ) -> Result { - let event_ids = capture - .event_ids - .iter() - .map(|event_id| normalize_text(event_id)) - .collect::>(); let events = llm_usage_events::table .filter(llm_usage_events::user_id.eq(user_id.to_string())) - .filter(llm_usage_events::event_id.eq_any(&event_ids)) + .filter(llm_usage_events::event_id.eq_any(&capture.event_ids)) .select(( llm_usage_events::id, llm_usage_events::device_context_id, @@ -512,23 +595,28 @@ pub(super) fn raw_events( user_id, from: query.from, to: query.to, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), + agent_names: &query.agent_name, + session_ids: &query.session_id, session_pk: query.session_pk, turn_index: query.turn_index, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), + llm_providers: &query.llm_provider, + llm_models: &query.llm_model, event_type: query.event_type.as_deref(), limit: page_size.saturating_add(1), offset, }, + query.includes_content(), )?; let has_next_page = events.len() > page_size_usize; if has_next_page { events.truncate(page_size_usize); } let devices = load_device_contexts(connection, &events)?; - let mut attachments = load_attachments(connection, &events)?; + let mut attachments = if query.includes_content() { + load_attachments(connection, &events)? + } else { + HashMap::new() + }; let responses = events .into_iter() .map(|event| { @@ -551,20 +639,24 @@ struct EventFilters<'a> { user_id: Uuid, from: Option>, to: Option>, - agent_name: Option<&'a str>, - session_id: Option<&'a str>, + /// Each list matches any of its values; an empty list matches everything. + agent_names: &'a [String], + session_ids: &'a [String], session_pk: Option, turn_index: Option, - llm_provider: Option<&'a str>, - llm_model: Option<&'a str>, + llm_providers: &'a [String], + llm_models: &'a [String], event_type: Option<&'a str>, limit: i64, offset: i64, } +/// Loads matching events newest first. Without content, text and metadata are +/// not read: they hold whole request bodies, and an event list never shows them. fn load_filtered_events( connection: &mut SqliteConnection, filters: &EventFilters<'_>, + include_content: bool, ) -> Result, AppError> { let mut query = llm_usage_events::table .filter(llm_usage_events::user_id.eq(filters.user_id.to_string())) @@ -575,12 +667,13 @@ fn load_filtered_events( if let Some(to) = filters.to { query = query.filter(llm_usage_events::observed_at.lt(timestamp_to_micros(to))); } - if let Some(agent_name) = filters.agent_name.and_then(non_empty) { - query = - query.filter(llm_usage_events::agent_name.eq_any(agent_name_filter_values(agent_name))); + let agent_names = exact_values_filter(filters.agent_names); + if !agent_names.is_empty() { + query = query.filter(llm_usage_events::agent_name.eq_any(agent_names)); } - if let Some(session_id) = filters.session_id.and_then(non_empty) { - query = query.filter(llm_usage_events::session_id.eq(session_id.trim())); + let session_ids = exact_values_filter(filters.session_ids); + if !session_ids.is_empty() { + query = query.filter(llm_usage_events::session_id.eq_any(session_ids)); } if let Some(session_pk) = filters.session_pk { query = query.filter(llm_usage_events::session_pk.eq(session_pk.to_string())); @@ -588,27 +681,62 @@ fn load_filtered_events( if let Some(turn_index) = filters.turn_index { query = query.filter(llm_usage_events::turn_index.eq(turn_index)); } - if let Some(provider) = filters.llm_provider.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_provider.eq(normalize_slug(provider))); + let providers = exact_values_filter(filters.llm_providers); + if !providers.is_empty() { + query = query.filter(llm_usage_events::llm_provider.eq_any(providers)); } - if let Some(model) = filters.llm_model.and_then(non_empty) { - query = query.filter(llm_usage_events::llm_model.eq(model.trim())); + let models = exact_values_filter(filters.llm_models); + if !models.is_empty() { + query = query.filter(llm_usage_events::llm_model.eq_any(models)); } if let Some(event_type) = filters.event_type.and_then(non_empty) { query = query.filter(llm_usage_events::event_type.eq(validate_event_type(event_type)?)); } - query + let query = query .order_by(( llm_usage_events::observed_at.desc(), llm_usage_events::id.desc(), )) .limit(filters.limit) - .offset(filters.offset) - .select(EventRow::as_select()) - .load::(connection)? - .into_iter() - .map(EventRow::into_record) - .collect() + .offset(filters.offset); + let rows = if include_content { + query + .select(EventRow::as_select()) + .load::(connection)? + } else { + query + .select(( + llm_usage_events::id, + llm_usage_events::user_id, + llm_usage_events::device_context_id, + llm_usage_events::session_pk, + llm_usage_events::turn_pk, + llm_usage_events::event_id, + llm_usage_events::agent_name, + llm_usage_events::agent_version, + llm_usage_events::session_id, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + None::.into_sql::>(), + None::>.into_sql::>(), + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + "{}".into_sql::(), + llm_usage_events::created_at, + llm_usage_events::stream_id, + llm_usage_events::response_id, + llm_usage_events::previous_response_id, + )) + .load::(connection)? + }; + rows.into_iter().map(EventRow::into_record).collect() } pub(super) fn session_timeline( @@ -616,22 +744,7 @@ pub(super) fn session_timeline( user_id: Uuid, session_pk: Uuid, ) -> Result { - let session = agent_sessions::table - .filter(agent_sessions::id.eq(session_pk.to_string())) - .filter(agent_sessions::user_id.eq(user_id.to_string())) - .select(SessionRow::as_select()) - .first::(connection) - .optional()? - .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))? - .into_record()?; - let turns = agent_turns::table - .filter(agent_turns::session_pk.eq(session.id.to_string())) - .order_by(agent_turns::turn_index.asc()) - .select(TurnRow::as_select()) - .load::(connection)? - .into_iter() - .map(TurnRow::into_record) - .collect::, _>>()?; + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; let mut events = llm_usage_events::table .filter(llm_usage_events::session_pk.eq(session.id.to_string())) .order_by(( @@ -660,6 +773,174 @@ pub(super) fn session_timeline( attachments.remove(&event_pk).unwrap_or_default(), )); } + let mut timeline = assemble_timeline(session, turns, events_by_turn); + session_titles::fill(connection, &mut timeline.session)?; + Ok(timeline) +} + +type OutlineRow = ( + String, + String, + String, + i32, + String, + String, + String, + i64, + i64, + i64, + i64, + i64, + i64, + i64, + i64, + Option, + bool, +); + +pub(super) fn session_outline( + connection: &mut SqliteConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result { + let (session, turns) = load_session_turns(connection, user_id, session_pk)?; + // Only fixed-size columns and SQL-computed summaries are selected, so the + // outline stays small no matter how large event text or metadata grows. + let rows = llm_usage_events::table + .filter(llm_usage_events::session_pk.eq(session.id.to_string())) + .select(( + llm_usage_events::id, + llm_usage_events::event_id, + llm_usage_events::turn_pk, + llm_usage_events::turn_index, + llm_usage_events::llm_provider, + llm_usage_events::llm_model, + llm_usage_events::event_type, + llm_usage_events::input_tokens, + llm_usage_events::output_tokens, + llm_usage_events::cache_read_tokens, + llm_usage_events::cache_write_tokens, + llm_usage_events::reasoning_tokens, + llm_usage_events::total_tokens, + llm_usage_events::observed_at, + sql::("coalesce(length(text), 0)"), + sql::>(&format!( + "CASE WHEN event_type = 'request' THEN substr(text, -{OUTLINE_PREVIEW_CHARS}) \ + ELSE substr(text, 1, {OUTLINE_PREVIEW_CHARS}) END" + )), + // SQLite text functions stop at the first NUL character, so the + // length and preview above are wrong for text containing NUL. + sql::("coalesce(instr(CAST(text AS BLOB), x'00') > 0, 0)"), + )) + .load::(connection)?; + let nul_text_ids = rows + .iter() + .filter(|row| row.16) + .map(|row| row.0.clone()) + .collect::>(); + let nul_texts = if nul_text_ids.is_empty() { + HashMap::new() + } else { + llm_usage_events::table + .filter(llm_usage_events::id.eq_any(nul_text_ids)) + .select((llm_usage_events::id, llm_usage_events::text)) + .load::<(String, Option)>(connection)? + .into_iter() + .filter_map(|(id, text)| text.map(|text| (id, text))) + .collect::>() + }; + let event_ids = rows.iter().map(|row| row.0.clone()).collect::>(); + let attachment_counts = llm_usage_event_attachments::table + .filter(llm_usage_event_attachments::event_pk.eq_any(event_ids)) + .group_by(llm_usage_event_attachments::event_pk) + .select((llm_usage_event_attachments::event_pk, count_star())) + .load::<(String, i64)>(connection)? + .into_iter() + .collect::>(); + let mut events = rows + .into_iter() + .map(|mut row| { + let attachment_count = attachment_counts.get(&row.0).copied().unwrap_or(0); + if let Some(text) = nul_texts.get(&row.0) { + (row.14, row.15) = text_outline(text, &row.6)?; + } + Ok(UsageEventOutline { + id: parse_uuid(&row.0, "event id")?, + event_id: row.1, + turn_pk: parse_uuid(&row.2, "event turn id")?, + turn_index: row.3, + llm_provider: row.4, + llm_model: row.5, + event_type: row.6, + input_tokens: row.7, + output_tokens: row.8, + cache_read_tokens: row.9, + cache_write_tokens: row.10, + reasoning_tokens: row.11, + total_tokens: row.12, + observed_at: timestamp_from_micros(row.13)?, + text_chars: row.14, + preview: row.15, + attachment_count, + }) + }) + .collect::, AppError>>()?; + UsageEventTimelineOrder::sort(&mut events); + let mut events_by_turn: HashMap> = HashMap::new(); + for event in events { + events_by_turn.entry(event.turn_pk).or_default().push(event); + } + let mut timeline = assemble_timeline(session, turns, events_by_turn); + session_titles::fill(connection, &mut timeline.session)?; + Ok(timeline) +} + +/// Computes the outline length and preview in Rust for text SQLite cannot measure. +fn text_outline(text: &str, event_type: &str) -> Result<(i64, Option), AppError> { + let chars = text.chars().count(); + let preview_chars = usize::try_from(OUTLINE_PREVIEW_CHARS) + .map_err(|error| AppError::internal(format!("invalid outline preview size: {error}")))?; + let preview = if event_type == "request" { + text.chars() + .skip(chars.saturating_sub(preview_chars)) + .collect() + } else { + text.chars().take(preview_chars).collect() + }; + let chars = i64::try_from(chars) + .map_err(|error| AppError::internal(format!("invalid event text length: {error}")))?; + Ok((chars, Some(preview))) +} + +fn load_session_turns( + connection: &mut SqliteConnection, + user_id: Uuid, + session_pk: Uuid, +) -> Result<(SessionRecord, Vec), AppError> { + let session = agent_sessions::table + .filter(agent_sessions::id.eq(session_pk.to_string())) + .filter(agent_sessions::user_id.eq(user_id.to_string())) + .select(SessionRow::as_select()) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("session not found: {session_pk}")))? + .into_record()?; + let turns = agent_turns::table + .filter(agent_turns::session_pk.eq(session.id.to_string())) + .order_by(agent_turns::turn_index.asc()) + .select(TurnRow::as_select()) + .load::(connection)? + .into_iter() + .map(TurnRow::into_record) + .collect::, _>>()?; + Ok((session, turns)) +} + +fn assemble_timeline( + session: SessionRecord, + turns: Vec, + mut events_by_turn: HashMap>, +) -> SessionTimelineResponse { let turn_timelines = turns .into_iter() .map(|turn| TurnTimeline { @@ -670,20 +951,49 @@ pub(super) fn session_timeline( events: events_by_turn.remove(&turn.id).unwrap_or_default(), }) .collect(); - Ok(SessionTimelineResponse { + SessionTimelineResponse { session: SessionInfo { session_pk: session.id, user_id: session.user_id, device_context_id: session.device_context_id, - agent_name: canonical_agent_name(&session.agent_name), + agent_name: session.agent_name, agent_version: session.agent_version, session_id: session.session_id, started_at: session.started_at, ended_at: session.ended_at, metadata: session.metadata, + title: None, + purpose: None, + parent_session_pk: None, }, turns: turn_timelines, - }) + } +} + +pub(super) fn usage_event( + connection: &mut SqliteConnection, + user_id: Uuid, + event_pk: Uuid, +) -> Result { + let event = llm_usage_events::table + .filter(llm_usage_events::id.eq(event_pk.to_string())) + .filter(llm_usage_events::user_id.eq(user_id.to_string())) + .select(EventRow::as_select()) + .first::(connection) + .optional()? + .ok_or_else(|| AppError::not_found(format!("usage event not found: {event_pk}")))? + .into_record()?; + let events = [event]; + let devices = load_device_contexts(connection, &events)?; + let mut attachments = load_attachments(connection, &events)?; + let [event] = events; + let event_pk = event.id; + let device_id = event.device_context_id; + Ok(event_response( + event, + devices.get(&device_id), + attachments.remove(&event_pk).unwrap_or_default(), + )) } pub(super) fn image_attachment( @@ -720,7 +1030,7 @@ pub(super) fn usage_summary( connection: &mut SqliteConnection, query: &SummaryQuery, user_id: Uuid, - scan_limit: i64, + default_limit: i64, ) -> Result { let group_by = parse_group_by(query.group_by.as_deref()); let dimensions = SummaryDimensions::from_group_by(&group_by); @@ -728,45 +1038,52 @@ pub(super) fn usage_summary( if !owner_matches_filter(user_id, &owner, query.user_filter.as_deref()) { return Ok(empty_summary(query, group_by)); } - let events = load_filtered_events( - connection, - &EventFilters { - user_id, - from: query.from, - to: query.to, - agent_name: query.agent_name.as_deref(), - session_id: query.session_id.as_deref(), - session_pk: None, - turn_index: None, - llm_provider: query.llm_provider.as_deref(), - llm_model: query.llm_model.as_deref(), - event_type: query.event_type.as_deref(), - limit: scan_limit, - offset: 0, - }, - )?; - let devices = load_device_contexts(connection, &events)?; - let mut buckets: HashMap = HashMap::new(); - for event in events { - let device = devices.get(&event.device_context_id); - let key = SummaryKey::new(&event, device, user_id, &owner, &dimensions); - buckets.entry(key).or_default().add(&event); - } - let mut rows = buckets + // Each list matches any of its values; an empty list matches everything. + let filter_list = |values: Vec| { + serde_json::to_string(&values) + .map(|json| (!values.is_empty(), json)) + .map_err(|error| AppError::internal(format!("serialize summary filter: {error}"))) + }; + let (has_agents, agent_names) = filter_list(exact_values_filter(&query.agent_name))?; + let (has_sessions, session_ids) = filter_list(exact_values_filter(&query.session_id))?; + let (has_providers, providers) = filter_list(exact_values_filter(&query.llm_provider))?; + let (has_models, models) = filter_list(exact_values_filter(&query.llm_model))?; + let event_type = query + .event_type + .as_deref() + .and_then(non_empty) + .map(validate_event_type) + .transpose()?; + let limit = normalize_limit(query.limit, default_limit, 100_000); + + // Bind order mirrors the numbered placeholders in the query. + let rows = sql_query(SUMMARY_AGGREGATE_SQL) + .bind::(dimensions.enabled(SummaryDimension::Day)) + .bind::(dimensions.enabled(SummaryDimension::Device)) + .bind::(dimensions.enabled(SummaryDimension::Agent)) + .bind::(dimensions.enabled(SummaryDimension::Provider)) + .bind::(dimensions.enabled(SummaryDimension::Model)) + .bind::(dimensions.enabled(SummaryDimension::EventType)) + .bind::(user_id.to_string()) + .bind::(query.from.map_or(i64::MIN, timestamp_to_micros)) + .bind::(query.to.map_or(i64::MAX, timestamp_to_micros)) + .bind::(has_agents) + .bind::(&agent_names) + .bind::(has_sessions) + .bind::(&session_ids) + .bind::(has_providers) + .bind::(&providers) + .bind::(has_models) + .bind::(&models) + .bind::(event_type.is_some()) + .bind::(event_type.unwrap_or_default()) + .bind::(limit) + .load::(connection)?; + let include_user = dimensions.enabled(SummaryDimension::User); + let rows = rows .into_iter() - .map(|(key, bucket)| bucket.into_row(key)) - .collect::>(); - rows.sort_by(|left, right| { - right - .total_tokens - .cmp(&left.total_tokens) - .then_with(|| left.agent_name.cmp(&right.agent_name)) - .then_with(|| left.user_email.cmp(&right.user_email)) - .then_with(|| left.day.cmp(&right.day)) - }); - let row_limit = usize::try_from(normalize_limit(query.limit, scan_limit, 100_000)) - .map_err(|error| AppError::internal(format!("invalid summary row limit: {error}")))?; - rows.truncate(row_limit); + .map(|row| row.into_summary_row(include_user.then_some((user_id, &owner)))) + .collect::, AppError>>()?; Ok(SummaryResponse { from: query.from, to: query.to, @@ -776,6 +1093,124 @@ pub(super) fn usage_summary( }) } +/// Usage totals grouped in SQL, reading only the columns a summary reports. +/// +/// Event text and metadata hold whole request bodies, so loading rows to sum +/// them in memory read gigabytes for a week of usage. The owner is the single +/// standalone user, so the user dimension is filled in afterwards. +/// +/// Bind order: `?1`..`?6` day, device, Agent, provider, model, and event-type +/// dimensions; `?7` owner; `?8`/`?9` observation bounds in microseconds, kept +/// unconditional so the owner-time index bounds the scan; `?10`..`?17` flags +/// and JSON value lists for the Agent, session, provider, and model filters, +/// each matching any listed value; `?18`/`?19` event-type filter; `?20` row +/// limit. +const SUMMARY_AGGREGATE_SQL: &str = "\ + SELECT \ + CASE WHEN ?1 THEN strftime('%Y-%m-%d', e.observed_at / 1000000, 'unixepoch') END AS day, \ + CASE WHEN ?2 THEN d.host_name END AS host_name, \ + CASE WHEN ?2 THEN d.platform END AS platform, \ + CASE WHEN ?2 THEN d.os_version END AS os_version, \ + CASE WHEN ?3 THEN e.agent_name END AS agent_name, \ + CASE WHEN ?4 THEN e.llm_provider END AS llm_provider, \ + CASE WHEN ?5 THEN e.llm_model END AS llm_model, \ + CASE WHEN ?6 THEN e.event_type END AS event_type, \ + COUNT(DISTINCT e.session_pk) AS sessions, \ + COUNT(DISTINCT e.turn_pk) AS turns, \ + COUNT(*) FILTER (WHERE e.event_type = 'request') AS requests, \ + COUNT(*) FILTER (WHERE e.event_type = 'response') AS responses, \ + COALESCE(SUM(e.input_tokens), 0) AS input_tokens, \ + COALESCE(SUM(e.output_tokens), 0) AS output_tokens, \ + COALESCE(SUM(e.cache_read_tokens), 0) AS cache_read_tokens, \ + COALESCE(SUM(e.cache_write_tokens), 0) AS cache_write_tokens, \ + COALESCE(SUM(e.reasoning_tokens), 0) AS reasoning_tokens, \ + COALESCE(SUM(e.total_tokens), 0) AS total_tokens \ + FROM llm_usage_events e \ + LEFT JOIN devices d ON ?2 AND d.id = e.device_context_id \ + WHERE e.user_id = ?7 \ + AND e.observed_at >= ?8 \ + AND e.observed_at < ?9 \ + AND (?10 = 0 OR e.agent_name IN (SELECT value FROM json_each(?11))) \ + AND (?12 = 0 OR e.session_id IN (SELECT value FROM json_each(?13))) \ + AND (?14 = 0 OR e.llm_provider IN (SELECT value FROM json_each(?15))) \ + AND (?16 = 0 OR e.llm_model IN (SELECT value FROM json_each(?17))) \ + AND (?18 = 0 OR e.event_type = ?19) \ + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8 \ + ORDER BY total_tokens DESC, agent_name ASC, day ASC \ + LIMIT ?20"; + +#[derive(QueryableByName)] +struct SummaryAggregateRow { + #[diesel(sql_type = Nullable)] + day: Option, + #[diesel(sql_type = Nullable)] + host_name: Option, + #[diesel(sql_type = Nullable)] + platform: Option, + #[diesel(sql_type = Nullable)] + os_version: Option, + #[diesel(sql_type = Nullable)] + agent_name: Option, + #[diesel(sql_type = Nullable)] + llm_provider: Option, + #[diesel(sql_type = Nullable)] + llm_model: Option, + #[diesel(sql_type = Nullable)] + event_type: Option, + #[diesel(sql_type = BigInt)] + sessions: i64, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + requests: i64, + #[diesel(sql_type = BigInt)] + responses: i64, + #[diesel(sql_type = BigInt)] + input_tokens: i64, + #[diesel(sql_type = BigInt)] + output_tokens: i64, + #[diesel(sql_type = BigInt)] + cache_read_tokens: i64, + #[diesel(sql_type = BigInt)] + cache_write_tokens: i64, + #[diesel(sql_type = BigInt)] + reasoning_tokens: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, +} + +impl SummaryAggregateRow { + fn into_summary_row(self, user: Option<(Uuid, &OwnerRecord)>) -> Result { + let count = |value: i64, name: &str| { + usize::try_from(value) + .map_err(|error| AppError::internal(format!("invalid summary {name}: {error}"))) + }; + Ok(SummaryRow { + day: self.day, + user_id: user.map(|(user_id, _)| user_id), + user_name: user.and_then(|(_, owner)| owner.name.clone()), + user_email: user.map(|(_, owner)| owner.email.clone()), + host_name: self.host_name, + platform: self.platform, + os_version: self.os_version, + agent_name: self.agent_name, + llm_provider: self.llm_provider, + llm_model: self.llm_model, + event_type: self.event_type, + sessions: count(self.sessions, "sessions")?, + turns: count(self.turns, "turns")?, + requests: self.requests, + responses: self.responses, + input_tokens: self.input_tokens, + output_tokens: self.output_tokens, + cache_read_tokens: self.cache_read_tokens, + cache_write_tokens: self.cache_write_tokens, + reasoning_tokens: self.reasoning_tokens, + total_tokens: self.total_tokens, + }) + } +} + struct OwnerRecord { email: String, name: Option, @@ -862,126 +1297,6 @@ impl SummaryDimensions { } } -#[derive(Hash, Eq, PartialEq)] -struct SummaryKey { - day: Option, - user_id: Option, - user_name: Option, - user_email: Option, - host_name: Option, - platform: Option, - os_version: Option, - agent_name: Option, - llm_provider: Option, - llm_model: Option, - event_type: Option, -} - -impl SummaryKey { - fn new( - event: &EventRecord, - device: Option<&DeviceRecord>, - user_id: Uuid, - owner: &OwnerRecord, - dimensions: &SummaryDimensions, - ) -> Self { - let include_user = dimensions.enabled(SummaryDimension::User); - let include_device = dimensions.enabled(SummaryDimension::Device); - Self { - day: dimensions - .enabled(SummaryDimension::Day) - .then(|| event.observed_at.format("%Y-%m-%d").to_string()), - user_id: include_user.then_some(user_id), - user_name: include_user.then(|| owner.name.clone()).flatten(), - user_email: include_user.then(|| owner.email.clone()), - host_name: include_device - .then(|| device.map(|value| value.host_name.clone())) - .flatten(), - platform: include_device - .then(|| device.map(|value| value.platform.clone())) - .flatten(), - os_version: include_device - .then(|| device.and_then(|value| value.os_version.clone())) - .flatten(), - agent_name: dimensions - .enabled(SummaryDimension::Agent) - .then(|| canonical_agent_name(&event.agent_name)), - llm_provider: dimensions - .enabled(SummaryDimension::Provider) - .then(|| event.llm_provider.clone()), - llm_model: dimensions - .enabled(SummaryDimension::Model) - .then(|| event.llm_model.clone()), - event_type: dimensions - .enabled(SummaryDimension::EventType) - .then(|| event.event_type.clone()), - } - } -} - -#[derive(Default)] -struct SummaryBucket { - sessions: HashSet, - turns: HashSet, - requests: i64, - responses: i64, - input_tokens: i64, - output_tokens: i64, - cache_read_tokens: i64, - cache_write_tokens: i64, - reasoning_tokens: i64, - total_tokens: i64, -} - -impl SummaryBucket { - fn add(&mut self, event: &EventRecord) { - self.sessions.insert(event.session_pk); - self.turns.insert(event.turn_pk); - self.requests = self - .requests - .saturating_add(i64::from(event.event_type == "request")); - self.responses = self - .responses - .saturating_add(i64::from(event.event_type == "response")); - self.input_tokens = self.input_tokens.saturating_add(event.input_tokens); - self.output_tokens = self.output_tokens.saturating_add(event.output_tokens); - self.cache_read_tokens = self - .cache_read_tokens - .saturating_add(event.cache_read_tokens); - self.cache_write_tokens = self - .cache_write_tokens - .saturating_add(event.cache_write_tokens); - self.reasoning_tokens = self.reasoning_tokens.saturating_add(event.reasoning_tokens); - self.total_tokens = self.total_tokens.saturating_add(event.total_tokens); - } - - fn into_row(self, key: SummaryKey) -> SummaryRow { - SummaryRow { - day: key.day, - user_id: key.user_id, - user_name: key.user_name, - user_email: key.user_email, - host_name: key.host_name, - platform: key.platform, - os_version: key.os_version, - agent_name: key.agent_name, - llm_provider: key.llm_provider, - llm_model: key.llm_model, - event_type: key.event_type, - sessions: self.sessions.len(), - turns: self.turns.len(), - requests: self.requests, - responses: self.responses, - input_tokens: self.input_tokens, - output_tokens: self.output_tokens, - cache_read_tokens: self.cache_read_tokens, - cache_write_tokens: self.cache_write_tokens, - reasoning_tokens: self.reasoning_tokens, - total_tokens: self.total_tokens, - } - } -} - fn load_device_contexts( connection: &mut SqliteConnection, events: &[EventRecord], @@ -1059,7 +1374,7 @@ fn event_response( platform: device.map(|value| value.platform.clone()), session_pk: event.session_pk, turn_pk: event.turn_pk, - agent_name: canonical_agent_name(&event.agent_name), + agent_name: event.agent_name, agent_version: event.agent_version, session_id: event.session_id, turn_index: event.turn_index, @@ -1075,20 +1390,14 @@ fn event_response( reasoning_tokens: event.reasoning_tokens, total_tokens: event.total_tokens, observed_at: event.observed_at, + stream_id: event.stream_id, + response_id: event.response_id, + previous_response_id: event.previous_response_id, metadata: event.metadata, attachments, } } -fn merge_json(stored: &str, incoming: &serde_json::Value, field: &str) -> Result { - let mut stored = serde_json::from_str::(stored) - .map_err(|error| AppError::internal(format!("stored {field} is invalid JSON: {error}")))?; - if let (Some(stored), Some(incoming)) = (stored.as_object_mut(), incoming.as_object()) { - stored.extend(incoming.clone()); - } - serialize_json(&stored, field) -} - fn serialize_json(value: &serde_json::Value, field: &str) -> Result { serde_json::to_string(value) .map_err(|error| AppError::internal(format!("serialize {field}: {error}"))) @@ -1100,15 +1409,25 @@ pub(super) const fn timestamp_to_micros(value: DateTime) -> i64 { #[cfg(test)] mod tests { - use diesel::{Connection, QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star, sql_query}; + use diesel::{ + Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection, dsl::count_star, + sql_query, + }; use serde_json::json; use uuid::Uuid; - use crate::{search::SessionSearchQuery, usage::IngestEventsRequest}; + use crate::{ + search::SessionSearchQuery, + usage::{IngestEventsRequest, RawEventsQuery, SummaryQuery}, + }; use super::{ - super::{migrations, schema::llm_usage_events, search}, - ingest_events, + super::{ + migrations, + schema::{agent_turns, llm_usage_events}, + search, + }, + ingest_events, raw_events, session_outline, usage_summary, }; const OWNER_ID: Uuid = Uuid::from_u128(1); @@ -1151,9 +1470,9 @@ mod tests { q: "searchable response".to_owned(), from: None, to: None, - agent_name: Some("codex".to_owned()), - llm_provider: None, - llm_model: None, + agent_name: vec!["claude-code".to_owned(), "codex".to_owned()], + llm_provider: Vec::new(), + llm_model: Vec::new(), event_type: None, page: None, page_size: None, @@ -1176,11 +1495,10 @@ mod tests { #[test] fn validation_failure_leaves_relational_and_fts_tables_empty() { let mut database = database(); - let mut request = request_with_text("invalid event"); - request.events[0].token_usage.input_tokens = Some(-1); + let request = request_with_text("invalid event"); - ingest_events(&mut database, &request, OWNER_ID, 100) - .expect_err("negative tokens should reject the batch"); + ingest_events(&mut database, &request, OWNER_ID, 0) + .expect_err("a batch over the size limit should be rejected"); let relational_count = llm_usage_events::table .select(count_star()) .first::(&mut database) @@ -1194,20 +1512,500 @@ mod tests { assert_eq!(fts_count, 0); } + #[test] + fn outline_measures_text_containing_nul_characters() { + let mut database = database(); + let text = format!("ab\0{}", "c".repeat(300)); + ingest_events(&mut database, &request_with_text(&text), OWNER_ID, 100) + .expect("ingest should succeed"); + let session_pk = llm_usage_events::table + .select(llm_usage_events::session_pk) + .first::(&mut database) + .expect("event should exist"); + let session_pk = Uuid::parse_str(&session_pk).expect("session pk should be a UUID"); + + let outline = + session_outline(&mut database, OWNER_ID, session_pk).expect("outline should load"); + + let event = &outline.turns[0].events[0]; + assert_eq!(event.text_chars, 303); + assert_eq!( + event.preview.as_deref(), + Some(format!("ab\0{}", "c".repeat(197)).as_str()) + ); + } + + #[test] + fn raw_body_archive_survives_ingest_retries_and_trajectory_projection() { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + use sha2::Digest as _; + let original = + b" {\"input\":[{\"role\":\"user\",\"content\":\"hello\"}],\"unknown\": [1,1]} \r\n"; + let mut database = database(); + let mut request = request_with_text("hello"); + request.events[0].event_type = crate::usage::UsageEventType::Request; + request.events[0].metadata["raw_body"] = json!({"transport":"http", "parts":[{ + "content_base64": STANDARD.encode(original), "byte_size": original.len(), "truncated": false, + "sha256": hex::encode(sha2::Sha256::digest(original)) + }]}); + let expected = request.events[0].metadata.clone(); + ingest_events(&mut database, &request, OWNER_ID, 100).unwrap(); + ingest_events(&mut database, &request, OWNER_ID, 100).unwrap(); + let session_pk = llm_usage_events::table + .select(llm_usage_events::session_pk) + .first::(&mut database) + .unwrap(); + let session_pk = Uuid::parse_str(&session_pk).unwrap(); + let timeline = super::session_timeline(&mut database, OWNER_ID, session_pk).unwrap(); + let stored = &timeline.turns[0].events[0].metadata; + assert_eq!(*stored, expected); + assert_eq!( + STANDARD + .decode( + stored["raw_body"]["parts"][0]["content_base64"] + .as_str() + .unwrap() + ) + .unwrap(), + original + ); + let trajectory = crate::usage::trajectory::build(timeline); + assert_eq!(trajectory.items[0].text.as_deref(), Some("hello")); + let after = super::session_timeline(&mut database, OWNER_ID, session_pk).unwrap(); + assert_eq!(after.turns[0].events.len(), 1); + assert_eq!(after.turns[0].events[0].metadata, expected); + } + + #[test] + fn ingest_stores_reported_values_unchanged() { + let mut database = database(); + let mut request = request_with_text(" hello \n"); + let event = &mut request.events[0]; + event.session_id = " Session ".to_owned(); + event.llm.provider = "Gateway_Host".to_owned(); + event.llm.model = " Model ".to_owned(); + event.device.platform = "macOS".to_owned(); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + let mut empty = request_with_text(""); + empty.events[0].event_id = "sqlite-empty-text".to_owned(); + ingest_events(&mut database, &empty, OWNER_ID, 100).expect("ingest should succeed"); + + let query: RawEventsQuery = + serde_json::from_value(json!({})).expect("query should deserialize"); + let events = raw_events(&mut database, &query, OWNER_ID, 50) + .expect("events should load") + .events; + let stored = events + .iter() + .find(|event| event.event_id == "sqlite-event") + .expect("the first event should be stored"); + assert_eq!(stored.session_id, " Session "); + assert_eq!(stored.llm_provider, "Gateway_Host"); + assert_eq!(stored.llm_model, " Model "); + assert_eq!(stored.platform.as_deref(), Some("macOS")); + assert_eq!(stored.text.as_deref(), Some(" hello \n")); + let empty = events + .iter() + .find(|event| event.event_id == "sqlite-empty-text") + .expect("the empty-text event should be stored"); + assert_eq!(empty.text.as_deref(), Some("")); + } + + #[test] + fn event_list_leaves_out_content_but_keeps_what_a_list_shows() { + let mut database = summary_seeded(); + let page = |database: &mut SqliteConnection, query: serde_json::Value| { + let query: RawEventsQuery = + serde_json::from_value(query).expect("query should deserialize"); + raw_events(database, &query, OWNER_ID, 50).expect("events should load") + }; + + let list = page(&mut database, json!({"fields": "list", "limit": 2_i64})); + let full = page(&mut database, json!({"limit": 2_i64})); + + assert_eq!(list.next_page_token.as_deref(), Some("2")); + assert_eq!(list.events.len(), 2); + for (listed, complete) in list.events.iter().zip(&full.events) { + assert_eq!(listed.id, complete.id); + assert_eq!(listed.total_tokens, complete.total_tokens); + assert_eq!(listed.host_name.as_deref(), Some("alice-mbp")); + assert_eq!(listed.text, None); + assert_eq!(listed.metadata, json!({})); + assert!(listed.attachments.is_empty()); + assert_eq!(complete.text.as_deref(), Some("body")); + assert_eq!(complete.metadata["raw_body"], json!("large")); + } + } + + fn summary_seeded() -> SqliteConnection { + let mut database = database(); + let event = |id: &str, agent: &str, session: &str, turn: i32, observed_at: &str| { + let (provider, model) = if agent == "codex" { + ("openai", "gpt-5") + } else { + ("anthropic", "claude-x") + }; + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos", "os_version": "15.1"}, + "agent": {"name": agent}, + "session_id": session, + "turn_id": format!("{session}-{turn}"), + "llm": {"provider": provider, "model": model}, + "event_type": if id.ends_with("req") { "request" } else { "response" }, + "text": "body", + "token_usage": if id.ends_with("req") { + json!({"input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 4_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 14_i64}) + } else { + json!({"input_tokens": 0_i64, "output_tokens": 5_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 5_i64}) + }, + "metadata": {"raw_body": "large"} + }) + }; + let request: IngestEventsRequest = serde_json::from_value(json!({ + "events": [ + event("a-req", "codex", "alpha", 1_i32, "2026-09-01T23:00:00Z"), + event("a-res", "codex", "alpha", 1_i32, "2026-09-01T23:30:00Z"), + event("b-req", "claude-desktop", "beta", 1_i32, "2026-09-02T01:00:00Z"), + event("c-res", "claude-code", "gamma", 1_i32, "2026-09-02T02:00:00Z"), + event("a2-req", "codex", "alpha", 2_i32, "2026-09-03T00:00:00Z"), + ] + })) + .expect("request should deserialize"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn summary(database: &mut SqliteConnection, query: serde_json::Value) -> serde_json::Value { + let query: SummaryQuery = serde_json::from_value(query).expect("query should deserialize"); + let response = + usage_summary(database, &query, OWNER_ID, 500).expect("summary should succeed"); + serde_json::to_value(response.rows).expect("rows should serialize") + } + + #[test] + fn summary_groups_in_sql_by_reported_agent_and_distinct_counts() { + let mut database = summary_seeded(); + + let rows = summary( + &mut database, + json!({"group_by": "agent,provider,model", "to": "2026-09-03T00:00:00Z"}), + ); + + // Claude Desktop is its own Agent, not an alias of Claude Code. + assert_eq!( + rows, + json!([ + { + "agent_name": "codex", "llm_provider": "openai", "llm_model": "gpt-5", + "sessions": 1_i64, "turns": 1_i64, "requests": 1_i64, "responses": 1_i64, + "input_tokens": 10_i64, "output_tokens": 5_i64, "cache_read_tokens": 4_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 19_i64 + }, + { + "agent_name": "claude-desktop", "llm_provider": "anthropic", + "llm_model": "claude-x", + "sessions": 1_i64, "turns": 1_i64, "requests": 1_i64, "responses": 0_i64, + "input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 4_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 14_i64 + }, + { + "agent_name": "claude-code", "llm_provider": "anthropic", + "llm_model": "claude-x", + "sessions": 1_i64, "turns": 1_i64, "requests": 0_i64, "responses": 1_i64, + "input_tokens": 0_i64, "output_tokens": 5_i64, "cache_read_tokens": 0_i64, + "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 5_i64 + } + ]) + ); + } + + #[test] + fn summary_filters_and_groups_by_utc_day_and_event_type() { + let mut database = summary_seeded(); + + let rows = summary( + &mut database, + json!({ + "group_by": "day,event_type", + "agent_name": ["claude-code", "claude-desktop"], + "limit": 10_i64, + }), + ); + + let mut buckets = rows + .as_array() + .expect("rows should be an array") + .iter() + .map(|row| { + ( + row["day"].clone(), + row["event_type"].clone(), + row["total_tokens"].clone(), + ) + }) + .collect::>(); + buckets.sort_by_key(|bucket| bucket.1.to_string()); + assert_eq!( + buckets, + vec![ + (json!("2026-09-02"), json!("request"), json!(14_i64)), + (json!("2026-09-02"), json!("response"), json!(5_i64)), + ] + ); + let codex_requests = summary( + &mut database, + json!({"group_by": "event_type", "agent_name": ["codex"], "event_type": "request"}), + ); + assert_eq!(codex_requests[0]["requests"], json!(2_i64)); + assert_eq!(codex_requests[0]["turns"], json!(2_i64)); + assert_eq!(codex_requests.as_array().map(Vec::len), Some(1)); + // Agent names match exactly: no alias expansion and no case folding. + for agent in ["claude", "Codex"] { + let rows = summary( + &mut database, + json!({"group_by": "agent", "agent_name": [agent]}), + ); + assert_eq!(rows, json!([]), "{agent} must not match another Agent"); + } + // Providers match exactly as reported, without case folding. + let folded = summary( + &mut database, + json!({"group_by": "agent", "llm_provider": ["OpenAI"]}), + ); + assert_eq!( + folded, + json!([]), + "OpenAI must not match the reported openai" + ); + // A list matches any of its values. + let either = summary( + &mut database, + json!({ + "group_by": "agent", + "llm_provider": ["openai", "anthropic"], + "session_id": ["alpha", "beta"], + }), + ); + let mut agents = either + .as_array() + .expect("rows should be an array") + .iter() + .map(|row| (row["agent_name"].clone(), row["sessions"].clone())) + .collect::>(); + agents.sort_by_key(|agent| agent.0.to_string()); + assert_eq!( + agents, + vec![ + (json!("claude-desktop"), json!(1_i64)), + (json!("codex"), json!(1_i64)), + ] + ); + } + + #[test] + fn summary_fills_owner_and_device_dimensions() { + let mut database = summary_seeded(); + + let rows = summary(&mut database, json!({"group_by": "user,device"})); + + assert_eq!(rows.as_array().map(Vec::len), Some(1)); + assert_eq!(rows[0]["user_id"], json!(OWNER_ID)); + assert!(rows[0]["user_email"].is_string()); + assert_eq!(rows[0]["host_name"], json!("alice-mbp")); + assert_eq!(rows[0]["platform"], json!("macos")); + assert_eq!(rows[0]["os_version"], json!("15.1")); + assert_eq!(rows[0]["sessions"], json!(3_i64)); + assert!(rows[0].get("agent_name").is_none()); + } + + /// One request observed `second` seconds into a fixed minute, working for + /// `turn` when it names one. + fn turn_event(id: &str, agent: &str, second: u32, turn: Option<&str>) -> serde_json::Value { + json!({ + "event_id": id, + "observed_at": format!("2026-10-08T00:00:{second:02}Z"), + "device": {"host_name": "local", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": "thread-session", + "turn_id": turn, + "llm": {"provider": "Anthropic", "model": "claude-x"}, + "event_type": "request", + "token_usage": {"input_tokens": 1_i64, "output_tokens": 0_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64}, + "metadata": {} + }) + } + + fn ingest_turn_events(database: &mut SqliteConnection, events: Vec) { + let request: IngestEventsRequest = + serde_json::from_value(json!({"events": serde_json::Value::Array(events)})) + .expect("request should deserialize"); + ingest_events(database, &request, OWNER_ID, 100).expect("ingest should succeed"); + } + + fn turn_of(database: &mut SqliteConnection, event_id: &str) -> i32 { + llm_usage_events::table + .filter(llm_usage_events::event_id.eq(event_id)) + .select(llm_usage_events::turn_index) + .first::(database) + .expect("event should be stored") + } + + /// Stored turns in number order: key, number, start and end second. + fn turns(database: &mut SqliteConnection) -> Vec<(Option, i32, i64, Option)> { + agent_turns::table + .order_by(agent_turns::turn_index.asc()) + .select(( + agent_turns::turn_key, + agent_turns::turn_index, + agent_turns::started_at, + agent_turns::ended_at, + )) + .load::<(Option, i32, i64, Option)>(database) + .expect("turns should load") + } + + fn second(value: i64) -> i64 { + super::timestamp_to_micros( + chrono::DateTime::parse_from_rfc3339(&format!("2026-10-08T00:00:{value:02}Z")) + .expect("timestamp should parse") + .into(), + ) + } + + #[test] + fn events_naming_a_turn_share_it_and_others_join_the_turn_in_progress() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![ + turn_event("main-1", "codex", 10, Some("p1")), + turn_event("subagent-1", "codex", 12, Some("p1")), + turn_event("recap", "codex", 20, None), + turn_event("main-2", "codex", 30, Some("p2")), + turn_event("subagent-2", "codex", 40, Some("p2")), + ], + ); + // Delivered late: a request naming no turn observed during turn 2, and + // a background agent of turn 1 still working during turn 2. + ingest_turn_events( + &mut database, + vec![ + turn_event("late-title", "codex", 32, None), + turn_event("late-subagent", "codex", 45, Some("p1")), + ], + ); + + for (event, turn) in [ + ("main-1", 1_i32), + ("subagent-1", 1_i32), + ("recap", 1_i32), + ("late-subagent", 1_i32), + ("main-2", 2_i32), + ("subagent-2", 2_i32), + ("late-title", 2_i32), + ] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } + assert_eq!( + turns(&mut database), + vec![ + (Some("p1".to_owned()), 1_i32, second(10), Some(second(45))), + (Some("p2".to_owned()), 2_i32, second(30), Some(second(40))), + ] + ); + } + + #[test] + fn the_first_named_turn_adopts_the_turn_opened_without_one() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![turn_event("probe", "claude-code", 5, None)], + ); + ingest_turn_events( + &mut database, + vec![ + turn_event("main-1", "claude-code", 10, Some("p1")), + turn_event("main-2", "claude-code", 30, Some("p2")), + // Earlier than every turn: joins the first one. + turn_event("early", "claude-code", 1, None), + ], + ); + + assert_eq!(turn_of(&mut database, "probe"), 1_i32); + assert_eq!(turn_of(&mut database, "main-1"), 1_i32); + assert_eq!(turn_of(&mut database, "early"), 1_i32); + assert_eq!(turn_of(&mut database, "main-2"), 2_i32); + assert_eq!( + turns(&mut database), + vec![ + (Some("p1".to_owned()), 1_i32, second(1), Some(second(10))), + (Some("p2".to_owned()), 2_i32, second(30), Some(second(30))), + ] + ); + } + + #[test] + fn turns_are_numbered_by_start_time_whatever_order_they_arrive_in() { + let mut database = database(); + ingest_turn_events( + &mut database, + vec![ + turn_event("b", "codex", 30, Some("b")), + turn_event("b-next", "codex", 31, Some("b")), + turn_event("c", "codex", 50, Some("c")), + ], + ); + assert_eq!(turn_of(&mut database, "c"), 2_i32); + + // A turn delivered late started first. + ingest_turn_events(&mut database, vec![turn_event("a", "codex", 10, Some("a"))]); + for (event, turn) in [("a", 1_i32), ("b", 2_i32), ("b-next", 2_i32), ("c", 3_i32)] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } + + // A late event of a known turn moves its start before every other turn. + ingest_turn_events( + &mut database, + vec![turn_event("c-early", "codex", 5, Some("c"))], + ); + for (event, turn) in [ + ("c-early", 1_i32), + ("c", 1_i32), + ("a", 2_i32), + ("b", 3_i32), + ("b-next", 3_i32), + ] { + assert_eq!(turn_of(&mut database, event), turn, "{event}"); + } + assert_eq!( + turns(&mut database) + .into_iter() + .map(|(key, index, ..)| (key, index)) + .collect::>(), + vec![ + (Some("c".to_owned()), 1_i32), + (Some("a".to_owned()), 2_i32), + (Some("b".to_owned()), 3_i32), + ] + ); + } + fn request_with_text(text: &str) -> IngestEventsRequest { serde_json::from_value(json!({ "events": [{ "event_id": "sqlite-event", "observed_at": "2026-08-27T00:00:00Z", "device": {"host_name": "local", "platform": "macos"}, - "agent": {"name": "Codex"}, + "agent": {"name": "codex"}, "session_id": "sqlite-session", - "turn_index": 1, - "llm": {"provider": "OpenAI", "model": "gpt-5"}, + "llm": {"provider": "openai", "model": "gpt-5"}, "event_type": "response", "text": text, - "token_usage": {"output_tokens": 3}, - "metadata": {"response_id": "sqlite-response"} + "token_usage": {"input_tokens": 0_i64, "output_tokens": 3_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 3_i64}, + "metadata": {} }] })) .expect("test ingest request should deserialize") diff --git a/crates/abyss-backend/src/storage/sqlite/schema.rs b/crates/abyss-backend/src/storage/sqlite/schema.rs index 80531fe..45eaec8 100644 --- a/crates/abyss-backend/src/storage/sqlite/schema.rs +++ b/crates/abyss-backend/src/storage/sqlite/schema.rs @@ -53,6 +53,7 @@ diesel::table! { user_id -> Text, session_pk -> Text, turn_index -> Integer, + turn_key -> Nullable, started_at -> BigInt, ended_at -> Nullable, created_at -> BigInt, @@ -88,6 +89,9 @@ diesel::table! { observed_at -> BigInt, metadata -> Text, created_at -> BigInt, + stream_id -> Nullable, + response_id -> Nullable, + previous_response_id -> Nullable, } } @@ -133,6 +137,21 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::{BigInt, Nullable, Text}; + + session_titles (session_pk) { + session_pk -> Text, + user_id -> Text, + title -> Nullable, + title_event_pk -> Nullable, + title_observed_at -> Nullable, + purpose -> Nullable, + updated_at -> BigInt, + parent_session_pk -> Nullable, + } +} + diesel::joinable!(agent_sessions -> app_users (user_id)); diesel::joinable!(agent_sessions -> devices (device_context_id)); diesel::joinable!(agent_turns -> agent_sessions (session_pk)); @@ -149,6 +168,7 @@ diesel::joinable!(agent_diagnostic_capture_events -> llm_usage_events (event_pk) diesel::joinable!(agent_diagnostic_captures -> agent_sessions (session_pk)); diesel::joinable!(agent_diagnostic_captures -> app_users (user_id)); diesel::joinable!(agent_diagnostic_captures -> devices (device_context_id)); +diesel::joinable!(session_titles -> agent_sessions (session_pk)); diesel::allow_tables_to_appear_in_same_query!( agent_diagnostic_capture_events, @@ -159,4 +179,5 @@ diesel::allow_tables_to_appear_in_same_query!( devices, llm_usage_events, llm_usage_event_attachments, + session_titles, ); diff --git a/crates/abyss-backend/src/storage/sqlite/search.rs b/crates/abyss-backend/src/storage/sqlite/search.rs index bdd4ead..2bc5c85 100644 --- a/crates/abyss-backend/src/storage/sqlite/search.rs +++ b/crates/abyss-backend/src/storage/sqlite/search.rs @@ -18,7 +18,6 @@ use crate::{ HIGHLIGHT_END, HIGHLIGHT_START, SearchFragment, SessionSearchMatch, SessionSearchQuery, SessionSearchResponse, SessionSearchResult, ValidatedSearchQuery, }, - usage::persistence::canonical_agent_name, }; use super::{ @@ -33,20 +32,31 @@ pub(super) fn session_search( query: SessionSearchQuery, ) -> Result { let query = query.validate()?; - let source = MatchSource::new(&query, user_id); - let total_sessions = count_sessions(connection, &source)?; - let session_ids = load_session_page(connection, &query, &source)?; - let matches = load_matches(connection, &session_ids, &source)?; + let hits = count_hits(connection, &query)?; + search(connection, user_id, query, hits) +} + +/// Runs a validated search; `hits` only chooses how observation times are read. +fn search( + connection: &mut SqliteConnection, + user_id: Uuid, + query: ValidatedSearchQuery, + hits: i64, +) -> Result { + let source = MatchSource::new(&query, user_id, hits); + let (page, total_sessions) = load_session_page(connection, &query, &source)?; + let session_ids = page + .iter() + .map(|(session_pk, _)| *session_pk) + .collect::>(); + let mut matches = load_matches(connection, &session_ids, &source)?; let mut details = load_session_details(connection, user_id, &session_ids)?; - let items = session_ids + let items = page .into_iter() - .filter_map(|session_pk| { + .filter_map(|(session_pk, match_count)| { let details = details.remove(&session_pk)?; - let session_matches = matches.get(&session_pk).cloned().unwrap_or_default(); - let match_count = session_matches - .first() - .map_or(0, |matched| matched.match_count); + let session_matches = matches.remove(&session_pk).unwrap_or_default(); let mut providers = session_matches .iter() .map(|matched| matched.llm_provider.clone()) @@ -64,7 +74,7 @@ pub(super) fn session_search( Some(SessionSearchResult { session_pk, session_id: details.session_id, - agent_name: canonical_agent_name(&details.agent_name), + agent_name: details.agent_name, agent_version: details.agent_version, host_name: details.host_name, platform: details.platform, @@ -91,62 +101,140 @@ pub(super) fn session_search( }) } +/// Matches the strongest events rank first: the lowest bm25, then the latest +/// observation, then the event key, so results are deterministic. +const MATCHES_PER_SESSION: i64 = 3; + #[derive(Clone)] enum FtsBind { Text(String), BigInt(i64), } +/// Above this many FTS hits, observation times come from the owner-time +/// covering index for all of the owner's events at once; below it, each hit +/// reads its own event row, which is cheaper than building that index. +const ROW_LOOKUP_MAX_HITS: i64 = 256; + +/// Where a page's session list is inserted into the shared expressions. +const SESSION_FILTER: &str = "/* session filter */"; + +/// Common table expressions that list every matching event of the owner with +/// its rank, observation time, and position and count within its session. +/// +/// Every step avoids reading wide rows: `hits` takes only rowids and bm25 from +/// the FTS index; event and session keys come from FTS5's content table rather +/// than through the virtual table, which would load each matching document. +/// Observation times sit after the event text in its row, so for many hits +/// they come from the owner-time covering index instead, and the time window +/// is applied after the join so SQLite keeps an index on that list. Event rows +/// are otherwise joined only for filters whose columns precede the text. struct MatchSource { sql: String, binds: Vec, } impl MatchSource { - fn new(query: &ValidatedSearchQuery, user_id: Uuid) -> Self { - let mut sql = String::from( - "FROM usage_events_fts - INNER JOIN llm_usage_events e ON e.id = usage_events_fts.event_pk - WHERE usage_events_fts MATCH ? AND e.user_id = ?", - ); - let mut binds = vec![ - FtsBind::Text(fts_query(&query.text)), - FtsBind::Text(user_id.to_string()), - ]; + fn new(query: &ValidatedSearchQuery, user_id: Uuid, hits: i64) -> Self { + let mut binds = vec![FtsBind::Text(fts_query(&query.text))]; + let read_rows = hits <= ROW_LOOKUP_MAX_HITS; + let (events_cte, joins, time_column) = if read_rows { + ( + "", + "INNER JOIN llm_usage_events e ON e.id = c.c0", + "e.observed_at", + ) + } else { + binds.push(FtsBind::Text(user_id.to_string())); + ( + ", + events AS MATERIALIZED ( + SELECT id, observed_at FROM llm_usage_events WHERE user_id = ? + )", + "INNER JOIN events ev ON ev.id = c.c0", + "ev.observed_at", + ) + }; + let mut conditions = String::from("1 = 1"); + if read_rows { + conditions.push_str(" AND e.user_id = ?"); + binds.push(FtsBind::Text(user_id.to_string())); + } if let Some(from) = query.from { - sql.push_str(" AND e.observed_at >= ?"); + conditions.push_str(" AND "); + conditions.push_str(time_column); + conditions.push_str(" >= ?"); binds.push(FtsBind::BigInt(timestamp_to_micros(from))); } if let Some(to) = query.to { - sql.push_str(" AND e.observed_at < ?"); + conditions.push_str(" AND "); + conditions.push_str(time_column); + conditions.push_str(" < ?"); binds.push(FtsBind::BigInt(timestamp_to_micros(to))); } - push_exact_filter( - &mut sql, - &mut binds, - "e.agent_name", - query.agent_name.as_deref(), - ); - push_exact_filter( - &mut sql, + let mut filters = String::new(); + push_any_filter(&mut filters, &mut binds, "e.agent_name", &query.agent_names); + push_any_filter( + &mut filters, &mut binds, "e.llm_provider", - query.llm_provider.as_deref(), + &query.llm_providers, ); + push_any_filter(&mut filters, &mut binds, "e.llm_model", &query.llm_models); push_exact_filter( - &mut sql, - &mut binds, - "e.llm_model", - query.llm_model.as_deref(), - ); - push_exact_filter( - &mut sql, + &mut filters, &mut binds, "e.event_type", query.event_type.as_deref(), ); + let event_join = if read_rows || filters.is_empty() { + "" + } else { + "INNER JOIN llm_usage_events e ON e.id = c.c0" + }; + let sql = format!( + "WITH hits AS MATERIALIZED ( + SELECT rowid AS fts_rowid, bm25(usage_events_fts) AS rank + FROM usage_events_fts + WHERE usage_events_fts MATCH ? + ){events_cte}, + matched AS MATERIALIZED ( + SELECT c.c0 AS event_pk, c.c2 AS session_pk, h.fts_rowid, h.rank, + {time_column} AS observed_at + FROM hits h + INNER JOIN usage_events_fts_content c ON c.id = h.fts_rowid + {joins} + {event_join} + WHERE {conditions}{filters} {SESSION_FILTER} + ), + ranked AS ( + SELECT matched.*, + ROW_NUMBER() OVER ( + PARTITION BY session_pk + ORDER BY rank ASC, observed_at DESC, event_pk ASC + ) AS position, + COUNT(*) OVER (PARTITION BY session_pk) AS match_count + FROM matched + )" + ); Self { sql, binds } } + + /// The expressions with matches limited to `session_ids`, or all sessions. + fn sql_for(&self, session_ids: &[Uuid]) -> (String, Vec) { + let mut binds = self.binds.clone(); + let session_filter = if session_ids.is_empty() { + String::new() + } else { + binds.extend( + session_ids + .iter() + .map(|session_pk| FtsBind::Text(session_pk.to_string())), + ); + format!("AND c.c2 IN ({})", vec!["?"; session_ids.len()].join(", ")) + }; + (self.sql.replace(SESSION_FILTER, &session_filter), binds) + } } fn push_exact_filter( @@ -163,6 +251,19 @@ fn push_exact_filter( } } +/// Matches any of `values`; an empty list adds no filter. +fn push_any_filter(sql: &mut String, binds: &mut Vec, column: &str, values: &[String]) { + if values.is_empty() { + return; + } + sql.push_str(" AND "); + sql.push_str(column); + sql.push_str(" IN ("); + sql.push_str(&vec!["?"; values.len()].join(", ")); + sql.push(')'); + binds.extend(values.iter().cloned().map(FtsBind::Text)); +} + fn bound_query(sql: String, binds: Vec) -> BoxedSqlQuery<'static, Sqlite, SqlQuery> { let mut query = sql_query(sql).into_boxed::(); for bind in binds { @@ -180,62 +281,80 @@ struct CountRow { count: i64, } -fn count_sessions( +/// FTS hits across all owners, before any filter; reads only the index. +fn count_hits( connection: &mut SqliteConnection, - source: &MatchSource, -) -> Result { - let sql = format!( - "SELECT COUNT(DISTINCT e.session_pk) AS count {}", - source.sql - ); - let count = bound_query(sql, source.binds.clone()) - .get_result::(connection)? - .count; - u64::try_from(count.max(0)) - .map_err(|error| AppError::internal(format!("invalid FTS session count: {error}"))) + query: &ValidatedSearchQuery, +) -> Result { + Ok(bound_query( + "SELECT COUNT(*) AS count FROM usage_events_fts WHERE usage_events_fts MATCH ?".to_owned(), + vec![FtsBind::Text(fts_query(&query.text))], + ) + .get_result::(connection)? + .count) } #[derive(QueryableByName)] -struct SessionIdRow { +struct SessionPageRow { #[diesel(sql_type = Text)] session_pk: String, + #[diesel(sql_type = BigInt)] + match_count: i64, + #[diesel(sql_type = BigInt)] + total_sessions: i64, } +/// One page of sessions with their match counts, ordered by each session's +/// strongest event, and the number of matching sessions. fn load_session_page( connection: &mut SqliteConnection, query: &ValidatedSearchQuery, source: &MatchSource, -) -> Result, AppError> { - let sql = format!( - "SELECT e.session_pk AS session_pk - {} - ORDER BY bm25(usage_events_fts) ASC, e.observed_at DESC, e.session_pk ASC", - source.sql +) -> Result<(Vec<(Uuid, u64)>, u64), AppError> { + let (sql, mut binds) = source.sql_for(&[]); + let page_sql = format!( + "{sql} + SELECT session_pk, match_count, COUNT(*) OVER () AS total_sessions + FROM ranked + WHERE position = 1 + ORDER BY rank ASC, observed_at DESC, session_pk ASC + LIMIT ? OFFSET ?" ); - let rows = bound_query(sql, source.binds.clone()).load::(connection)?; - let page_start = usize::try_from(query.offset) - .map_err(|error| AppError::internal(format!("invalid FTS page offset: {error}")))?; - let page_size = usize::try_from(query.page_size) - .map_err(|error| AppError::internal(format!("invalid FTS page size: {error}")))?; - let page_end = page_start.saturating_add(page_size); - let mut seen = HashSet::new(); - let mut sessions = Vec::with_capacity(page_size); - for row in rows { - let session_pk = parse_uuid(&row.session_pk, "FTS session id")?; - if !seen.insert(session_pk) { - continue; - } - let position = seen.len().saturating_sub(1); - if position >= page_start { - sessions.push(session_pk); + binds.push(FtsBind::BigInt(i64::from(query.page_size))); + binds.push(FtsBind::BigInt(i64::from(query.offset))); + let rows = bound_query(page_sql, binds).load::(connection)?; + let total_sessions = match rows.first() { + Some(row) => row.total_sessions, + // A page past the end has no rows to carry the total. + None if query.offset > 0 => { + let (sql, binds) = source.sql_for(&[]); + bound_query( + format!("{sql} SELECT COUNT(*) AS count FROM ranked WHERE position = 1"), + binds, + ) + .get_result::(connection)? + .count } - if seen.len() >= page_end { - break; - } - } - Ok(sessions) + None => 0, + }; + let page = rows + .into_iter() + .map(|row| { + Ok(( + parse_uuid(&row.session_pk, "FTS session id")?, + u64::try_from(row.match_count.max(0)).map_err(|error| { + AppError::internal(format!("invalid FTS match count: {error}")) + })?, + )) + }) + .collect::, AppError>>()?; + let total_sessions = u64::try_from(total_sessions.max(0)) + .map_err(|error| AppError::internal(format!("invalid FTS session count: {error}")))?; + Ok((page, total_sessions)) } +/// The strongest matches of each session, with highlighted fragments built +/// only for the events returned. fn load_matches( connection: &mut SqliteConnection, session_ids: &[Uuid], @@ -244,61 +363,44 @@ fn load_matches( if session_ids.is_empty() { return Ok(HashMap::new()); } - let selected_placeholders = std::iter::repeat_n("?", session_ids.len()) - .collect::>() - .join(","); - let sql = format!( - "SELECT - e.id AS event_pk, - e.session_pk AS session_pk, + let (sql, mut binds) = source.sql_for(session_ids); + let matches_sql = format!( + "{sql}, + top AS MATERIALIZED ( + SELECT * FROM ranked WHERE position <= {MATCHES_PER_SESSION} + ) + SELECT + top.event_pk AS event_pk, + top.session_pk AS session_pk, e.turn_pk AS turn_pk, e.turn_index AS turn_index, e.event_type AS event_type, e.llm_provider AS llm_provider, e.llm_model AS llm_model, - e.observed_at AS observed_at, + top.observed_at AS observed_at, snippet(usage_events_fts, 4, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS content_fragment, - snippet(usage_events_fts, 7, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) - AS command_fragment, - snippet(usage_events_fts, 8, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) - AS path_fragment, snippet(usage_events_fts, 5, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS tool_name_fragment, snippet(usage_events_fts, 6, '{HIGHLIGHT_START}', '{HIGHLIGHT_END}', '…', 32) AS tool_content_fragment - {} - AND e.session_pk IN ({selected_placeholders}) - ORDER BY bm25(usage_events_fts) ASC, e.observed_at DESC, e.id ASC", - source.sql + FROM top + INNER JOIN usage_events_fts ON usage_events_fts.rowid = top.fts_rowid + INNER JOIN llm_usage_events e ON e.id = top.event_pk + WHERE usage_events_fts MATCH ? + ORDER BY top.session_pk, top.position" ); - let mut binds = source.binds.clone(); - binds.extend( - session_ids - .iter() - .map(|session_pk| FtsBind::Text(session_pk.to_string())), - ); - let rows = bound_query(sql, binds).load::(connection)?; - let mut by_session: HashMap)> = HashMap::new(); + binds.push(source.binds[0].clone()); + let rows = bound_query(matches_sql, binds).load::(connection)?; + let mut by_session: HashMap> = HashMap::new(); for row in rows { let matched = row.into_matched()?; - let entry = by_session + by_session .entry(matched.session_pk) - .or_insert_with(|| (0, Vec::new())); - entry.0 = entry.0.saturating_add(1); - if entry.1.len() < 3 { - entry.1.push(matched); - } + .or_default() + .push(matched); } - Ok(by_session - .into_iter() - .map(|(session_pk, (match_count, mut matches))| { - for matched in &mut matches { - matched.match_count = match_count; - } - (session_pk, matches) - }) - .collect()) + Ok(by_session) } #[derive(QueryableByName)] @@ -322,10 +424,6 @@ struct MatchedEventRow { #[diesel(sql_type = Nullable)] content_fragment: Option, #[diesel(sql_type = Nullable)] - command_fragment: Option, - #[diesel(sql_type = Nullable)] - path_fragment: Option, - #[diesel(sql_type = Nullable)] tool_name_fragment: Option, #[diesel(sql_type = Nullable)] tool_content_fragment: Option, @@ -335,8 +433,6 @@ impl MatchedEventRow { fn into_matched(self) -> Result { let fragments = [ self.content_fragment, - self.command_fragment, - self.path_fragment, self.tool_name_fragment, self.tool_content_fragment, ] @@ -354,13 +450,11 @@ impl MatchedEventRow { llm_provider: self.llm_provider, llm_model: self.llm_model, observed_at: timestamp_from_micros(self.observed_at)?, - match_count: 0, fragments, }) } } -#[derive(Clone)] struct MatchedEvent { event_pk: Uuid, session_pk: Uuid, @@ -370,7 +464,6 @@ struct MatchedEvent { llm_provider: String, llm_model: String, observed_at: DateTime, - match_count: u64, fragments: Vec, } @@ -475,7 +568,21 @@ fn fts_query(value: &str) -> String { #[cfg(test)] mod tests { - use super::fts_query; + use diesel::{Connection, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use crate::{ + search::{SessionSearchQuery, SessionSearchResponse}, + usage::IngestEventsRequest, + }; + + use super::{ + super::{migrations, repository::ingest_events}, + ROW_LOOKUP_MAX_HITS, fts_query, search, + }; + + const OWNER_ID: Uuid = Uuid::from_u128(1); #[test] fn fts_query_uses_literal_and_terms() { @@ -485,4 +592,153 @@ mod tests { ); assert_eq!(fts_query(r#"quoted"value"#), r#""quoted""value""#); } + + fn event(id: &str, session: &str, observed_at: &str, text: &str) -> Value { + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos"}, + "agent": {"name": "codex"}, + "session_id": session, + "turn_index": 1, + "llm": {"provider": "openai", "model": "gpt-5"}, + "event_type": "response", + "text": text, + "token_usage": {"input_tokens": 0_i64, "output_tokens": 1_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64}, + "metadata": {} + }) + } + + /// Three sessions: `alpha` has five equal hits, `beta` one strong hit, and + /// `gamma` one hit diluted by other words. + fn seeded() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + let mut events = (1_u32..=5_u32) + .map(|minute| { + event( + &format!("a{minute}"), + "alpha", + &format!("2026-09-01T00:0{minute}:00Z"), + "needle note", + ) + }) + .collect::>(); + events.push(event("b1", "beta", "2026-09-02T00:00:00Z", "needle")); + events.push(event( + "g1", + "gamma", + "2026-09-03T00:00:00Z", + "needle among many other unrelated words in a much longer reply", + )); + events.push(event("z1", "zeta", "2026-09-04T00:00:00Z", "nothing here")); + let request: IngestEventsRequest = + serde_json::from_value(json!({ "events": events })).expect("request should parse"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn run(database: &mut SqliteConnection, query: Value, hits: i64) -> SessionSearchResponse { + let query: SessionSearchQuery = + serde_json::from_value(query).expect("query should deserialize"); + search( + database, + OWNER_ID, + query.validate().expect("query should validate"), + hits, + ) + .expect("search should succeed") + } + + fn summary(response: &SessionSearchResponse) -> Vec<(String, u64, usize)> { + response + .items + .iter() + .map(|item| { + ( + item.session_id.clone(), + item.match_count, + item.matches.len(), + ) + }) + .collect() + } + + #[test] + fn sessions_rank_by_their_strongest_event_and_page_in_order() { + let mut database = seeded(); + + let first = run(&mut database, json!({"q": "needle", "page_size": 2_u32}), 0); + let second = run( + &mut database, + json!({"q": "needle", "page": 2_u32, "page_size": 2_u32}), + 0, + ); + let past_end = run( + &mut database, + json!({"q": "needle", "page": 5_u32, "page_size": 2_u32}), + 0, + ); + + assert_eq!(first.total_sessions, 3); + assert!(first.has_more); + assert_eq!( + summary(&first), + [("beta".to_owned(), 1, 1), ("alpha".to_owned(), 5, 3)] + ); + assert_eq!(summary(&second), [("gamma".to_owned(), 1, 1)]); + assert!(!second.has_more); + assert_eq!(past_end.total_sessions, 3); + assert!(past_end.items.is_empty()); + } + + #[test] + fn a_session_returns_its_three_latest_equal_matches_and_counts_all() { + let mut database = seeded(); + + let response = run(&mut database, json!({"q": "needle note"}), 0); + + assert_eq!(summary(&response), [("alpha".to_owned(), 5, 3)]); + let minutes = response.items[0] + .matches + .iter() + .map(|matched| matched.observed_at.format("%M").to_string()) + .collect::>(); + // Equal scores fall back to the latest observation first. + assert_eq!(minutes, ["05", "04", "03"]); + assert!( + response.items[0].matches[0].fragments[0] + .segments + .iter() + .any(|segment| segment.highlighted) + ); + } + + #[test] + fn both_time_lookups_return_the_same_results_with_filters() { + let mut database = seeded(); + for query in [ + json!({"q": "needle"}), + json!({"q": "needle", "from": "2026-09-01T00:03:00Z", "to": "2026-09-02T12:00:00Z"}), + json!({"q": "needle", "agent_name": ["codex"], "event_type": "response", "page_size": 1_u32}), + json!({"q": "needle", "llm_model": ["absent"]}), + ] { + let by_rows = run(&mut database, query.clone(), 0); + let by_index = run(&mut database, query.clone(), ROW_LOOKUP_MAX_HITS + 1); + assert_eq!( + serde_json::to_value(&by_rows).expect("serializes"), + serde_json::to_value(&by_index).expect("serializes"), + "{query}" + ); + } + let windowed = run( + &mut database, + json!({"q": "needle", "from": "2026-09-01T00:03:00Z", "to": "2026-09-02T12:00:00Z"}), + ROW_LOOKUP_MAX_HITS + 1, + ); + assert_eq!( + summary(&windowed), + [("beta".to_owned(), 1, 1), ("alpha".to_owned(), 3, 3)] + ); + } } diff --git a/crates/abyss-backend/src/storage/sqlite/session_list.rs b/crates/abyss-backend/src/storage/sqlite/session_list.rs new file mode 100644 index 0000000..74390a9 --- /dev/null +++ b/crates/abyss-backend/src/storage/sqlite/session_list.rs @@ -0,0 +1,362 @@ +//! SQLite session index ordered by latest activity. + +use diesel::{ + QueryableByName, RunQueryDsl, SqliteConnection, sql_query, + sql_types::{BigInt, Bool, Nullable, Text}, +}; +use uuid::Uuid; + +use crate::{ + error::AppError, + usage::session_list::{SessionListItem, SessionListResponse, ValidatedSessionListQuery}, +}; + +use super::{ + models::{parse_uuid, timestamp_from_micros}, + repository::timestamp_to_micros, +}; + +/// Per-session activity with the filters that apply before paging. +/// +/// Bind order: `?1` owner, `?2`/`?3` agent filter flag and JSON name list, +/// `?4`/`?5` lower activity bound, `?6`/`?7` upper start bound, `?8`/`?9` +/// JSON provider list and `?10`/`?11` JSON model list a session must have used +/// at least one of. +const ACTIVITY_CTE: &str = "\ + WITH activity AS ( \ + SELECT \ + s.id AS session_pk, \ + s.session_id AS session_id, \ + s.agent_name AS agent_name, \ + s.agent_version AS agent_version, \ + d.host_name AS host_name, \ + d.platform AS platform, \ + s.started_at AS started_at, \ + s.ended_at AS ended_at, \ + COALESCE(MAX(e.observed_at), s.started_at) AS last_activity_at, \ + (SELECT COUNT(*) FROM agent_turns t WHERE t.session_pk = s.id) AS turns, \ + COUNT(e.session_pk) AS events, \ + COALESCE(SUM(e.total_tokens), 0) AS total_tokens, \ + st.title AS title, \ + st.purpose AS purpose \ + FROM agent_sessions s \ + JOIN devices d ON d.id = s.device_context_id \ + LEFT JOIN session_titles st ON st.session_pk = s.id \ + LEFT JOIN llm_usage_events e ON e.session_pk = s.id \ + WHERE s.user_id = ?1 \ + AND (?2 = 0 OR s.agent_name IN (SELECT value FROM json_each(?3))) \ + AND (?8 = 0 OR EXISTS ( \ + SELECT 1 FROM llm_usage_events p \ + WHERE p.session_pk = s.id \ + AND p.llm_provider IN (SELECT value FROM json_each(?9)))) \ + AND (?10 = 0 OR EXISTS ( \ + SELECT 1 FROM llm_usage_events m \ + WHERE m.session_pk = s.id \ + AND m.llm_model IN (SELECT value FROM json_each(?11)))) \ + GROUP BY s.id \ + ), \ + filtered AS ( \ + SELECT * FROM activity \ + WHERE (?4 = 0 OR last_activity_at >= ?5) \ + AND (?6 = 0 OR started_at < ?7) \ + ) "; + +#[derive(QueryableByName)] +struct SessionListRow { + #[diesel(sql_type = Text)] + session_pk: String, + #[diesel(sql_type = Text)] + session_id: String, + #[diesel(sql_type = Text)] + agent_name: String, + #[diesel(sql_type = Nullable)] + agent_version: Option, + #[diesel(sql_type = Text)] + host_name: String, + #[diesel(sql_type = Text)] + platform: String, + #[diesel(sql_type = BigInt)] + started_at: i64, + #[diesel(sql_type = Nullable)] + ended_at: Option, + #[diesel(sql_type = BigInt)] + last_activity_at: i64, + #[diesel(sql_type = BigInt)] + turns: i64, + #[diesel(sql_type = BigInt)] + events: i64, + #[diesel(sql_type = BigInt)] + total_tokens: i64, + #[diesel(sql_type = Nullable)] + title: Option, + #[diesel(sql_type = Nullable)] + purpose: Option, +} + +#[derive(QueryableByName)] +struct CountRow { + #[diesel(sql_type = BigInt)] + total: i64, +} + +pub(super) fn list_sessions( + connection: &mut SqliteConnection, + user_id: Uuid, + query: &ValidatedSessionListQuery, +) -> Result { + let agent_names = json_list(&query.agent_names, "agent")?; + let providers = json_list(&query.llm_providers, "provider")?; + let models = json_list(&query.llm_models, "model")?; + let has_agents = !query.agent_names.is_empty(); + let from = query.from.map_or(0, timestamp_to_micros); + let to = query.to.map_or(0, timestamp_to_micros); + + let total = sql_query(format!( + "{ACTIVITY_CTE} SELECT COUNT(*) AS total FROM filtered" + )) + .bind::(user_id.to_string()) + .bind::(has_agents) + .bind::(&agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(!query.llm_providers.is_empty()) + .bind::(&providers) + .bind::(!query.llm_models.is_empty()) + .bind::(&models) + .get_result::(connection)? + .total; + + let rows = sql_query(format!( + "{ACTIVITY_CTE} SELECT * FROM filtered \ + ORDER BY last_activity_at DESC, session_pk DESC LIMIT ?12 OFFSET ?13" + )) + .bind::(user_id.to_string()) + .bind::(has_agents) + .bind::(&agent_names) + .bind::(query.from.is_some()) + .bind::(from) + .bind::(query.to.is_some()) + .bind::(to) + .bind::(!query.llm_providers.is_empty()) + .bind::(&providers) + .bind::(!query.llm_models.is_empty()) + .bind::(&models) + .bind::(i64::from(query.page_size)) + .bind::(query.offset) + .load::(connection)?; + + let items = rows + .into_iter() + .map(|row| { + Ok(SessionListItem { + session_pk: parse_uuid(&row.session_pk, "session id")?, + session_id: row.session_id, + agent_name: row.agent_name, + agent_version: row.agent_version, + host_name: row.host_name, + platform: row.platform, + started_at: timestamp_from_micros(row.started_at)?, + ended_at: row.ended_at.map(timestamp_from_micros).transpose()?, + last_activity_at: timestamp_from_micros(row.last_activity_at)?, + turns: row.turns, + events: row.events, + total_tokens: row.total_tokens, + title: row.title, + purpose: row.purpose, + }) + }) + .collect::, AppError>>()?; + Ok(SessionListResponse::new(query, total, items)) +} + +fn json_list(values: &[String], filter: &str) -> Result { + serde_json::to_string(values) + .map_err(|error| AppError::internal(format!("serialize {filter} filter: {error}"))) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use diesel::{Connection, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use crate::usage::{IngestEventsRequest, session_list::SessionListQuery}; + + use super::{super::migrations, super::repository::ingest_events, list_sessions}; + + const OWNER_ID: Uuid = Uuid::from_u128(1); + + fn database() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + database + } + + fn event(id: &str, session: &str, agent: &str, observed_at: &str, turn: i32) -> Value { + event_with_model(id, session, agent, observed_at, turn, "gpt-5") + } + + fn event_with_model( + id: &str, + session: &str, + agent: &str, + observed_at: &str, + turn: i32, + model: &str, + ) -> Value { + json!({ + "event_id": id, + "observed_at": observed_at, + "device": {"host_name": "alice-mbp", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": session, + "turn_id": format!("{session}-{turn}"), + "llm": {"provider": "OpenAI", "model": model}, + "event_type": "response", + "text": "reply", + "token_usage": {"input_tokens": 0_i64, "output_tokens": 10_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 10_i64}, + "metadata": {} + }) + } + + fn seeded() -> SqliteConnection { + let mut database = database(); + let request: IngestEventsRequest = serde_json::from_value(json!({ + "events": [ + event("a1", "alpha", "codex", "2026-09-01T00:00:00Z", 1), + event_with_model("a2", "alpha", "codex", "2026-09-03T00:00:00Z", 2, "gpt-5-mini"), + event("b1", "beta", "claude-desktop", "2026-09-02T00:00:00Z", 1), + event("c1", "gamma", "codex", "2026-08-01T00:00:00Z", 1), + ] + })) + .expect("request should deserialize"); + ingest_events(&mut database, &request, OWNER_ID, 100).expect("ingest should succeed"); + database + } + + fn list(database: &mut SqliteConnection, query: SessionListQuery) -> Vec<(String, i64, i64)> { + let query = query.validate().expect("query should validate"); + list_sessions(database, OWNER_ID, &query) + .expect("sessions should list") + .items + .into_iter() + .map(|item| (item.session_id, item.turns, item.events)) + .collect() + } + + #[test] + fn lists_sessions_by_latest_activity_with_counts() { + let mut database = seeded(); + let query = SessionListQuery::default().validate().expect("valid"); + let response = list_sessions(&mut database, OWNER_ID, &query).expect("should list"); + + assert_eq!(response.total_sessions, 3); + assert!(!response.has_more); + let alpha = &response.items[0]; + assert_eq!(alpha.session_id, "alpha"); + assert_eq!((alpha.turns, alpha.events, alpha.total_tokens), (2, 2, 20)); + assert_eq!(alpha.host_name, "alice-mbp"); + assert_eq!( + alpha.last_activity_at, + Utc.with_ymd_and_hms(2026, 9, 3, 0, 0, 0).unwrap() + ); + let order: Vec<_> = response + .items + .iter() + .map(|item| item.session_id.as_str()) + .collect(); + assert_eq!(order, ["alpha", "beta", "gamma"]); + } + + #[test] + fn filters_by_exact_agent_and_activity_window() { + let mut database = seeded(); + let desktop = SessionListQuery { + agent_name: vec!["claude-desktop".to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, desktop), [("beta".to_owned(), 1, 1)]); + // Claude Desktop is not an alias of Claude Code, and case is not folded. + for agent in ["claude-code", "Codex"] { + let other = SessionListQuery { + agent_name: vec![agent.to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, other), Vec::new(), "{agent}"); + } + let either = SessionListQuery { + agent_name: vec!["claude-desktop".to_owned(), "codex".to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, either).len(), 3); + + // gamma went quiet before the window; alpha is still active inside it. + let window = SessionListQuery { + from: Some(Utc.with_ymd_and_hms(2026, 8, 15, 0, 0, 0).unwrap()), + to: Some(Utc.with_ymd_and_hms(2026, 9, 2, 12, 0, 0).unwrap()), + ..SessionListQuery::default() + }; + let ids: Vec<_> = list(&mut database, window) + .into_iter() + .map(|row| row.0) + .collect(); + assert_eq!(ids, ["alpha", "beta"]); + } + + #[test] + fn filters_by_models_or_providers_the_session_used() { + let mut database = seeded(); + let mini = SessionListQuery { + llm_model: vec!["gpt-5-mini".to_owned(), "unused-model".to_owned()], + ..SessionListQuery::default() + }; + let ids: Vec<_> = list(&mut database, mini) + .into_iter() + .map(|row| row.0) + .collect(); + assert_eq!(ids, ["alpha"]); + + let openai = SessionListQuery { + llm_provider: vec!["anthropic".to_owned(), "OpenAI".to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, openai).len(), 3); + + let other = SessionListQuery { + llm_provider: vec!["anthropic".to_owned()], + ..SessionListQuery::default() + }; + assert_eq!(list(&mut database, other), Vec::new()); + } + + #[test] + fn pages_through_sessions() { + let mut database = seeded(); + let first = SessionListQuery { + page_size: Some(2), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + let page = list_sessions(&mut database, OWNER_ID, &first).expect("should list"); + assert_eq!( + (page.items.len(), page.total_sessions, page.has_more), + (2, 3, true) + ); + + let second = SessionListQuery { + page: Some(2), + page_size: Some(2), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + let page = list_sessions(&mut database, OWNER_ID, &second).expect("should list"); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].session_id, "gamma"); + assert!(!page.has_more); + } +} diff --git a/crates/abyss-backend/src/storage/sqlite/session_titles.rs b/crates/abyss-backend/src/storage/sqlite/session_titles.rs new file mode 100644 index 0000000..86ec7e8 --- /dev/null +++ b/crates/abyss-backend/src/storage/sqlite/session_titles.rs @@ -0,0 +1,435 @@ +//! SQLite persistence of session names reported by the collector. + +use chrono::Utc; +use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; +use serde_json::Value; +use uuid::Uuid; + +use super::{ + models::parse_uuid, + schema::{llm_usage_events, session_titles}, +}; +use crate::{ + error::AppError, + usage::{ + SessionInfo, + session_title::{TITLE_GENERATION_PURPOSE, TitleContribution, default_title}, + }, +}; + +/// Stored title, purpose, and matched task session of one session. +type StoredTitle = (Option, Option, Option); + +/// One stored event considered for title capture. +pub(super) struct CapturedEvent<'a> { + pub(super) user_id: Uuid, + pub(super) session_pk: Uuid, + pub(super) event_pk: Uuid, + pub(super) event_type: &'a str, + pub(super) metadata: &'a Value, + pub(super) naming_thread: bool, + pub(super) observed_at: i64, +} + +/// Records what a newly stored event says about its session's name. +/// +/// An Agent title replaces a default title and an older Agent title. A default +/// title keeps the earliest user input, even after late delivery, and never +/// replaces an Agent title. +pub(super) fn capture( + connection: &mut SqliteConnection, + event: &CapturedEvent<'_>, +) -> Result<(), AppError> { + let session = event.session_pk.to_string(); + let agent_title = + match TitleContribution::of(event.event_type, event.metadata, event.naming_thread) { + None => return Ok(()), + Some(TitleContribution::TitleThread) => { + return mark_title_thread(connection, &session, event.user_id); + } + Some(TitleContribution::Agent(title)) => Some(title), + Some(TitleContribution::UserInput) => None, + }; + let stored = session_titles::table + .find(&session) + .select(( + session_titles::title_event_pk, + session_titles::title_observed_at, + )) + .first::<(Option, Option)>(connection) + .optional()? + .and_then(|(event_pk, observed_at)| event_pk.zip(observed_at)); + let agent = agent_title.is_some(); + let replaces = match stored { + None => true, + Some((_, stored_at)) if agent && stored_at < event.observed_at => true, + Some((_, stored_at)) if !agent && stored_at <= event.observed_at => false, + // Otherwise only a stored default title gives way. + Some((event_pk, _)) => !is_agent_title(connection, &event_pk)?, + }; + if !replaces { + return Ok(()); + } + // Read the user input only once it could name the session. + let Some(title) = agent_title + .map(str::to_owned) + .or_else(|| default_title(event.metadata)) + else { + return Ok(()); + }; + let now = Utc::now().timestamp_micros(); + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(&session), + session_titles::user_id.eq(event.user_id.to_string()), + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk.to_string()), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::title.eq(&title), + session_titles::title_event_pk.eq(event.event_pk.to_string()), + session_titles::title_observed_at.eq(event.observed_at), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + Ok(()) +} + +/// Agent titles are read from responses and default titles from requests, so +/// the side of the event that gave the stored title tells them apart. +fn is_agent_title(connection: &mut SqliteConnection, event_pk: &str) -> Result { + Ok(llm_usage_events::table + .find(event_pk) + .select(llm_usage_events::event_type) + .first::(connection) + .optional()? + .is_some_and(|event_type| event_type == "response")) +} + +/// Marks a session as a thread that names another session. +fn mark_title_thread( + connection: &mut SqliteConnection, + session: &str, + user_id: Uuid, +) -> Result<(), AppError> { + let now = Utc::now().timestamp_micros(); + diesel::insert_into(session_titles::table) + .values(( + session_titles::session_pk.eq(session), + session_titles::user_id.eq(user_id.to_string()), + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .on_conflict(session_titles::session_pk) + .do_update() + .set(( + session_titles::purpose.eq(TITLE_GENERATION_PURPOSE), + session_titles::updated_at.eq(now), + )) + .execute(connection)?; + Ok(()) +} + +/// Adds the stored title, purpose, and matched task session to a session. +pub(super) fn fill( + connection: &mut SqliteConnection, + session: &mut SessionInfo, +) -> Result<(), AppError> { + (session.title, session.purpose, session.parent_session_pk) = + load(connection, session.session_pk)?; + Ok(()) +} + +/// Returns the stored title and purpose of one session, and for a naming +/// thread the task session it was matched to. +fn load(connection: &mut SqliteConnection, session_pk: Uuid) -> Result { + let Some((title, purpose, parent)) = session_titles::table + .find(session_pk.to_string()) + .select(( + session_titles::title, + session_titles::purpose, + session_titles::parent_session_pk, + )) + .first::<(Option, Option, Option)>(connection) + .optional()? + else { + return Ok((None, None, None)); + }; + let parent = parent + .map(|parent| parse_uuid(&parent, "parent session id")) + .transpose()?; + Ok((title, purpose, parent)) +} +#[cfg(test)] +mod tests { + use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; + use serde_json::{Value, json}; + use uuid::Uuid; + + use super::{ + super::{migrations, repository::ingest_events, schema::agent_sessions}, + load, + }; + use crate::usage::IngestEventsRequest; + + const OWNER_ID: Uuid = Uuid::from_u128(1); + + fn database() -> SqliteConnection { + let mut database = SqliteConnection::establish(":memory:").expect("SQLite should open"); + migrations::run(&mut database).expect("migration should succeed"); + database + } + + /// A request whose collector conversation holds one user message. + fn request(agent: &str, session: &str, id: &str, at: &str, input: &str) -> Value { + event( + agent, + session, + id, + at, + "request", + json!({"conversation": { + "messages": [{"role": "user", "text": input}], + "completeness": "complete" + }}), + ) + } + + /// The response of a naming call, carrying the collector's title. + fn titled_response(session: &str, id: &str, at: &str, title: &str) -> Value { + event( + "claude-code", + session, + id, + at, + "response", + json!({"purpose": "session_title", "session_title": title}), + ) + } + + fn event(agent: &str, session: &str, id: &str, at: &str, kind: &str, metadata: Value) -> Value { + let mut event = json!({ + "event_id": id, + "observed_at": at, + "device": {"host_name": "local", "platform": "macos"}, + "agent": {"name": agent}, + "session_id": session, + "llm": {"provider": "anthropic", "model": "claude-haiku"}, + "event_type": kind, + "token_usage": {"input_tokens": 0_i64, "output_tokens": 1_i64, "cache_read_tokens": 0_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 1_i64} + }); + event["metadata"] = metadata; + event + } + + fn ingest(database: &mut SqliteConnection, events: &[Value]) { + let request: IngestEventsRequest = + serde_json::from_value(json!({ "events": events })).expect("request should parse"); + ingest_events(database, &request, OWNER_ID, 100).expect("ingest should succeed"); + } + + fn session_title( + database: &mut SqliteConnection, + session: &str, + ) -> (Option, Option) { + let pk = agent_sessions::table + .filter(agent_sessions::session_id.eq(session)) + .select(agent_sessions::id) + .first::(database) + .expect("session should exist"); + let (title, purpose, _) = + load(database, Uuid::parse_str(&pk).expect("uuid")).expect("title should load"); + (title, purpose) + } + + #[test] + fn names_a_session_from_the_collector_title() { + let mut database = database(); + let mut naming = request( + "claude-code", + "cc", + "r2", + "2026-09-30T00:00:02Z", + "fix the build", + ); + naming["metadata"]["purpose"] = json!("session_title"); + ingest( + &mut database, + &[ + request( + "claude-code", + "cc", + "r1", + "2026-09-30T00:00:00Z", + "fix the build", + ), + naming, + titled_response("cc", "s2", "2026-09-30T00:00:03Z", "Build fix"), + ], + ); + + assert_eq!( + session_title(&mut database, "cc"), + (Some("Build fix".to_owned()), None) + ); + } + + #[test] + fn agent_titles_outrank_default_titles_in_any_arrival_order() { + for title_first in [false, true] { + let mut database = database(); + let mut events = vec![ + request( + "claude-code", + "cc", + "r1", + "2026-09-30T00:00:00Z", + "fix the build", + ), + titled_response("cc", "s1", "2026-09-30T00:00:03Z", "Build fix"), + ]; + // An earlier user request delivered late does not replace the title. + if title_first { + events.reverse(); + } + for value in events { + ingest(&mut database, &[value]); + } + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Build fix"), + "title first: {title_first}" + ); + } + } + + #[test] + fn keeps_the_newest_agent_title() { + let mut database = database(); + ingest( + &mut database, + &[ + titled_response("cc", "s2", "2026-09-30T02:00:01Z", "Newer"), + titled_response("cc", "s1", "2026-09-30T01:00:01Z", "Older"), + ], + ); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Newer") + ); + ingest( + &mut database, + &[titled_response( + "cc", + "s3", + "2026-09-30T03:00:01Z", + "Newest", + )], + ); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Newest") + ); + } + + #[test] + fn title_like_replies_without_the_collector_field_are_ordinary_output() { + let mut database = database(); + let mut reply = event( + "claude-code", + "cc", + "s1", + "2026-09-30T00:00:01Z", + "response", + json!({"conversation": { + "messages": [{"role": "assistant", "text": "{\"title\":\"Quoted\"}"}], + "completeness": "complete" + }}), + ); + reply["text"] = json!("{\"title\":\"Quoted\"}"); + ingest( + &mut database, + &[ + request("claude-code", "cc", "r1", "2026-09-30T00:00:00Z", "Name it"), + reply, + ], + ); + assert_eq!( + session_title(&mut database, "cc").0.as_deref(), + Some("Name it") + ); + } + + #[test] + fn default_titles_keep_the_earliest_user_request_even_when_delivered_late() { + let mut database = database(); + for (id, at, kind, input, expected) in [ + ("r2", "2026-09-30T00:00:02Z", "request", "Second", "Second"), + ("r3", "2026-09-30T00:00:03Z", "request", "Third", "Second"), + ("r1", "2026-09-30T00:00:01Z", "request", "First", "First"), + ( + "s0", + "2026-09-30T00:00:00Z", + "response", + "Response", + "First", + ), + ] { + let mut value = request("cursor", "default", id, at, input); + value["event_type"] = json!(kind); + ingest(&mut database, &[value]); + assert_eq!( + session_title(&mut database, "default").0.as_deref(), + Some(expected) + ); + } + } + + #[test] + fn codex_task_sessions_keep_their_first_input_and_naming_threads_are_marked() { + let mut database = database(); + let mut naming = request( + "codex", + "title-thread", + "n1", + "2026-09-30T00:00:01Z", + "Generate a concise, single-line task title.\nUser prompt:\n画一只鹈鹕", + ); + naming["naming_thread"] = json!(true); + let task = request( + "codex", + "task", + "t1", + "2026-09-30T00:00:00Z", + "…", + ); + ingest( + &mut database, + &[ + task, + naming, + event( + "codex", + "title-thread", + "n2", + "2026-09-30T00:00:02Z", + "response", + json!({}), + ), + ], + ); + + assert_eq!( + session_title(&mut database, "task").0.as_deref(), + Some("…") + ); + assert_eq!( + session_title(&mut database, "title-thread"), + (None, Some("title_generation".to_owned())) + ); + } +} diff --git a/crates/abyss-backend/src/usage/conversation.rs b/crates/abyss-backend/src/usage/conversation.rs new file mode 100644 index 0000000..9814625 --- /dev/null +++ b/crates/abyss-backend/src/usage/conversation.rs @@ -0,0 +1,84 @@ +//! Typed access to collector-produced conversation projections; no provider decoding. + +use serde::Deserialize; +use serde_json::Value; + +#[derive(Deserialize)] +pub struct Conversation { + pub messages: Vec, + /// Whether the first message is the request's own instructions, such as + /// a system prompt or OpenAI `instructions`; absent means none. + #[serde(default)] + pub instructions_present: bool, + completeness: Completeness, +} + +#[derive(Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +enum Completeness { + Complete, + Partial, + Unsupported, +} + +/// One message as the collector reports it; its role is kept as reported. +#[derive(Clone, Deserialize)] +pub struct Message { + #[serde(default)] + pub id: Option, + pub role: String, + pub text: String, +} + +impl Conversation { + /// The completely decoded conversation of an event, as the `AgentEvent` + /// contract defines it. + pub fn read(metadata: &Value) -> Option { + let value = metadata.get("conversation")?; + let conversation: Self = Self::deserialize(value).ok()?; + (conversation.completeness == Completeness::Complete).then_some(conversation) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roles_are_kept_as_reported() { + let metadata = serde_json::json!({"conversation": { + "messages": [ + {"role": "reasoning", "text": "plan"}, + {"role": "some_new_role", "text": "other"}, + {"role": "user", "text": "hello"} + ], + "instructions_present": false, + "completeness": "complete" + }}); + let conversation = Conversation::read(&metadata).expect("complete conversation"); + assert_eq!( + conversation + .messages + .iter() + .map(|message| message.role.as_str()) + .collect::>(), + ["reasoning", "some_new_role", "user"] + ); + } + + #[test] + fn conversations_outside_the_contract_are_not_read() { + let messages = serde_json::json!([{"role": "user", "text": "hello"}]); + for conversation in [ + serde_json::json!({"messages": messages, "instructions_present": false}), + serde_json::json!({ + "messages": messages, + "instructions_present": false, + "completeness": "partial" + }), + ] { + let metadata = serde_json::json!({"conversation": conversation}); + assert!(Conversation::read(&metadata).is_none()); + } + } +} diff --git a/crates/abyss-backend/src/usage/event_order.rs b/crates/abyss-backend/src/usage/event_order.rs index 5969899..4d4421e 100644 --- a/crates/abyss-backend/src/usage/event_order.rs +++ b/crates/abyss-backend/src/usage/event_order.rs @@ -1,21 +1,13 @@ //! Deterministic ordering for normalized events inside Agent timeline turns. //! -//! Codex emits a provider response chain through `response_id` and -//! `previous_response_id`. Timeline ordering prefers that native relationship, -//! then falls back to high-precision observation time, event side, and stable -//! event identity. - -use std::{ - cmp::Ordering, - collections::{BTreeMap, HashMap, HashSet}, -}; +//! Events are ordered by turn, high-precision observation time, event side, +//! and stable event identity. use chrono::{DateTime, Utc}; /// Event fields required by deterministic timeline ordering. pub trait TimelineEvent { fn turn_index(&self) -> i32; - fn metadata(&self) -> &serde_json::Value; fn observed_at(&self) -> DateTime; fn event_type(&self) -> &str; fn event_id(&self) -> &str; @@ -27,8 +19,22 @@ impl TimelineEvent for crate::db::models::UsageEvent { self.turn_index } - fn metadata(&self) -> &serde_json::Value { - &self.metadata + fn observed_at(&self) -> DateTime { + self.observed_at + } + + fn event_type(&self) -> &str { + &self.event_type + } + + fn event_id(&self) -> &str { + &self.event_id + } +} + +impl TimelineEvent for super::UsageEventOutline { + fn turn_index(&self) -> i32 { + self.turn_index } fn observed_at(&self) -> DateTime { @@ -48,426 +54,136 @@ impl TimelineEvent for crate::db::models::UsageEvent { pub struct UsageEventTimelineOrder; impl UsageEventTimelineOrder { - /// Sorts events by turn, provider chain, observation time, side, and ID. + /// Sorts events by turn, observation time, side, and ID. /// /// The final event identifier tie-breaker makes output deterministic even - /// for corrupt cycles or collectors with identical timestamps. + /// for collectors with identical timestamps. pub fn sort(events: &mut [T]) { - let provider_ranks = ProviderResponseRanks::from_events(events); events.sort_by(|left, right| { - left.turn_index().cmp(&right.turn_index()).then_with(|| { - provider_ranks.compare_events(left, right).then_with(|| { - left.observed_at() - .cmp(&right.observed_at()) - .then_with(|| { - event_side_rank(left.event_type()) - .cmp(&event_side_rank(right.event_type())) - }) - .then_with(|| left.event_id().cmp(right.event_id())) + left.turn_index() + .cmp(&right.turn_index()) + .then_with(|| left.observed_at().cmp(&right.observed_at())) + .then_with(|| { + event_side_rank(left.event_type()).cmp(&event_side_rank(right.event_type())) }) - }) + .then_with(|| left.event_id().cmp(right.event_id())) }); } } -struct ProviderResponseRanks { - ranks: HashMap<(i32, String), usize>, - native_turns: HashSet, -} - -impl ProviderResponseRanks { - fn from_events(events: &[T]) -> Self { - let mut nodes_by_turn = BTreeMap::>::new(); - // Native provider ordering is safe only when every event in a turn has - // a response_id. A partially instrumented turn falls back as a whole so - // ranked and unranked events cannot interleave unpredictably. - let mut native_turns = events - .iter() - .map(TimelineEvent::turn_index) - .collect::>(); - for event in events { - let Some(response_id) = metadata_string(event.metadata(), "response_id") else { - native_turns.remove(&event.turn_index()); - continue; - }; - let previous_response_id = - metadata_string(event.metadata(), "previous_response_id").map(str::to_owned); - let nodes = nodes_by_turn.entry(event.turn_index()).or_default(); - let node = - nodes - .entry(response_id.to_owned()) - .or_insert_with(|| ProviderResponseNode { - response_id: response_id.to_owned(), - previous_response_id: previous_response_id.clone(), - first_observed_at: event.observed_at(), - first_event_id: event.event_id().to_owned(), - }); - node.merge_evidence(event, previous_response_id); - } - - let mut ranks = HashMap::new(); - for (turn_index, nodes) in nodes_by_turn { - Self::rank_turn(turn_index, &nodes, &mut ranks); - } - Self { - ranks, - native_turns, - } +const fn event_side_rank(event_type: &str) -> u8 { + match event_type.as_bytes() { + b"request" => 0, + b"response" => 1, + _ => 2, } +} - fn rank_turn( - turn_index: i32, - nodes: &HashMap, - ranks: &mut HashMap<(i32, String), usize>, - ) { - let mut roots = Vec::new(); - let mut children = HashMap::>::new(); - for node in nodes.values() { - match node - .previous_response_id - .as_ref() - .filter(|previous| previous.as_str() != node.response_id) - .filter(|previous| nodes.contains_key(previous.as_str())) - { - Some(previous) => children - .entry(previous.clone()) - .or_default() - .push(node.response_id.clone()), - None => roots.push(node.response_id.clone()), - } - } - - // Branches can occur after retries or malformed evidence. Sort roots and - // siblings by stable observed/event fallback data before traversal. - let compare_ids = |left: &String, right: &String| { - nodes - .get(left) - .expect("provider response id should resolve") - .fallback_cmp( - nodes - .get(right) - .expect("provider response id should resolve"), - ) - }; - roots.sort_by(compare_ids); - for child_ids in children.values_mut() { - child_ids.sort_by(compare_ids); - } - - let mut visited = HashSet::new(); - let mut next_rank = 0_usize; - for root in roots { - Self::rank_chain( - turn_index, - root, - &children, - ranks, - &mut visited, - &mut next_rank, - ); - } +#[cfg(test)] +mod tests { + use chrono::{DateTime, TimeZone as _, Timelike as _, Utc}; - // Cycles have no root. Traversing remaining nodes after rooted chains - // guarantees every response still receives a deterministic rank. - let mut remaining = nodes - .keys() - .filter(|response_id| !visited.contains(response_id.as_str())) - .cloned() - .collect::>(); - remaining.sort_by(compare_ids); - for response_id in remaining { - Self::rank_chain( - turn_index, - response_id, - &children, - ranks, - &mut visited, - &mut next_rank, - ); - } - } + use super::{TimelineEvent, UsageEventTimelineOrder}; - fn rank_chain( + struct Sample { turn_index: i32, - root: String, - children: &HashMap>, - ranks: &mut HashMap<(i32, String), usize>, - visited: &mut HashSet, - next_rank: &mut usize, - ) { - let mut pending = vec![root]; - while let Some(response_id) = pending.pop() { - if !visited.insert(response_id.clone()) { - continue; - } - ranks.insert((turn_index, response_id.clone()), *next_rank); - *next_rank = next_rank.saturating_add(1); - if let Some(child_ids) = children.get(&response_id) { - pending.extend(child_ids.iter().rev().cloned()); - } - } + observed_at: DateTime, + event_type: &'static str, + event_id: &'static str, } - fn compare_events(&self, left: &T, right: &T) -> Ordering { - if left.turn_index() != right.turn_index() - || !self.native_turns.contains(&left.turn_index()) - { - return Ordering::Equal; + impl TimelineEvent for Sample { + fn turn_index(&self) -> i32 { + self.turn_index } - let left_response_id = metadata_string(left.metadata(), "response_id"); - let right_response_id = metadata_string(right.metadata(), "response_id"); - - // Request and response observations that describe the same provider - // call remain adjacent and request-first regardless of timestamp ties. - if left_response_id.is_some() && left_response_id == right_response_id { - return event_side_rank(left.event_type()).cmp(&event_side_rank(right.event_type())); + fn observed_at(&self) -> DateTime { + self.observed_at } - let native_order = left_response_id - .and_then(|response_id| self.ranks.get(&(left.turn_index(), response_id.to_owned()))) - .zip(right_response_id.and_then(|response_id| { - self.ranks - .get(&(right.turn_index(), response_id.to_owned())) - })) - .map(|(left_rank, right_rank)| left_rank.cmp(right_rank)); - if let Some(ordering) = native_order.filter(|ordering| !ordering.is_eq()) { - return ordering; + fn event_type(&self) -> &str { + self.event_type } - Ordering::Equal - } -} - -struct ProviderResponseNode { - response_id: String, - previous_response_id: Option, - first_observed_at: DateTime, - first_event_id: String, -} - -impl ProviderResponseNode { - fn merge_evidence( - &mut self, - event: &T, - previous_response_id: Option, - ) { - if self.previous_response_id.is_none() { - self.previous_response_id = previous_response_id; - } - if (event.observed_at(), event.event_id()) - < (self.first_observed_at, self.first_event_id.as_str()) - { - self.first_observed_at = event.observed_at(); - event.event_id().clone_into(&mut self.first_event_id); + fn event_id(&self) -> &str { + self.event_id } } - fn fallback_cmp(&self, other: &Self) -> Ordering { - self.first_observed_at - .cmp(&other.first_observed_at) - .then_with(|| self.first_event_id.cmp(&other.first_event_id)) - .then_with(|| self.response_id.cmp(&other.response_id)) + fn timestamp() -> DateTime { + Utc.with_ymd_and_hms(2026, 8, 4, 2, 12, 24) + .single() + .expect("timestamp should be valid") } -} -fn metadata_string<'a>(metadata: &'a serde_json::Value, key: &str) -> Option<&'a str> { - metadata - .get(key) - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) -} - -const fn event_side_rank(event_type: &str) -> u8 { - match event_type.as_bytes() { - b"request" => 0, - b"response" => 1, - _ => 2, + fn event( + turn_index: i32, + observed_at: DateTime, + event_type: &'static str, + event_id: &'static str, + ) -> Sample { + Sample { + turn_index, + observed_at, + event_type, + event_id, + } } -} - -#[cfg(all(test, feature = "postgres-es"))] -mod tests { - use chrono::{TimeZone as _, Timelike as _, Utc}; - use serde_json::json; - use uuid::Uuid; - - use super::UsageEventTimelineOrder; - use crate::db::models::UsageEvent; - - #[test] - fn native_response_chain_precedes_timestamps_and_collector_ordinals() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event( - "b-response", - "response", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 1_i64}), - ), - usage_event( - "a-response", - "response", - timestamp, - json!({"response_id": "resp_a", "provider_call_index": 2_i64}), - ), - usage_event( - "b-request", - "request", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 1_i64}), - ), - usage_event( - "a-request", - "request", - timestamp, - json!({"response_id": "resp_a", "provider_call_index": 2_i64}), - ), - ]; + fn sorted_ids(mut events: Vec) -> Vec<&'static str> { UsageEventTimelineOrder::sort(&mut events); - - assert_eq!( - event_ids(&events), - ["a-request", "a-response", "b-request", "b-response"] - ); + events.iter().map(|event| event.event_id).collect() } #[test] - fn timestamp_precedes_collector_ordinal_when_native_chain_is_incomplete() { - let first_timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let second_timestamp = first_timestamp + chrono::TimeDelta::microseconds(1); - let mut events = vec![ - usage_event( - "later-despite-smaller-ordinal", - "response", - second_timestamp, - json!({"provider_call_index": 1_i64, "response_id": "resp_later"}), - ), - usage_event( - "earlier-despite-larger-ordinal", - "response", - first_timestamp, - json!({"provider_call_index": 2_i64}), - ), - ]; - - UsageEventTimelineOrder::sort(&mut events); - + fn turn_precedes_observation_time() { + let first = timestamp(); + let second = first + chrono::TimeDelta::microseconds(1); assert_eq!( - event_ids(&events), - [ - "earlier-despite-larger-ordinal", - "later-despite-smaller-ordinal" - ] + sorted_ids(vec![ + event(2, first, "request", "turn-2"), + event(1, second, "request", "turn-1"), + ]), + ["turn-1", "turn-2"] ); } #[test] - fn high_precision_timestamp_orders_legacy_events_without_provider_evidence() { - let first = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid") + fn high_precision_timestamp_orders_events_within_a_turn() { + let first = timestamp() .with_nanosecond(1_000) .expect("microsecond should be valid"); let second = first .with_nanosecond(2_000) .expect("microsecond should be valid"); - let mut events = vec![ - usage_event("second", "request", second, json!({})), - usage_event("first", "response", first, json!({})), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["first", "second"]); + assert_eq!( + sorted_ids(vec![ + event(1, second, "request", "second"), + event(1, first, "response", "first"), + ]), + ["first", "second"] + ); } #[test] - fn request_precedes_response_for_legacy_timestamp_ties() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event("response", "response", timestamp, json!({})), - usage_event("request", "request", timestamp, json!({})), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["request", "response"]); + fn request_precedes_response_for_timestamp_ties() { + assert_eq!( + sorted_ids(vec![ + event(1, timestamp(), "response", "response"), + event(1, timestamp(), "request", "request"), + ]), + ["request", "response"] + ); } #[test] - fn broken_cycles_fall_back_to_deterministic_event_identity() { - let timestamp = Utc - .with_ymd_and_hms(2026, 8, 4, 2, 12, 24) - .single() - .expect("timestamp should be valid"); - let mut events = vec![ - usage_event( - "second", - "response", - timestamp, - json!({"response_id": "resp_b", "previous_response_id": "resp_a", "provider_call_index": 2_i64}), - ), - usage_event( - "first", - "response", - timestamp, - json!({"response_id": "resp_a", "previous_response_id": "resp_b", "provider_call_index": 1_i64}), - ), - ]; - - UsageEventTimelineOrder::sort(&mut events); - - assert_eq!(event_ids(&events), ["first", "second"]); - } - - fn usage_event( - event_id: &str, - event_type: &str, - observed_at: chrono::DateTime, - metadata: serde_json::Value, - ) -> UsageEvent { - UsageEvent { - id: Uuid::nil(), - user_id: Uuid::nil(), - device_context_id: Uuid::nil(), - session_pk: Uuid::nil(), - turn_pk: Uuid::nil(), - event_id: event_id.to_owned(), - agent_name: "codex".to_owned(), - agent_version: None, - session_id: "session".to_owned(), - turn_index: 1, - llm_provider: "openai".to_owned(), - llm_model: "gpt-test".to_owned(), - event_type: event_type.to_owned(), - text: None, - text_sha256: None, - input_tokens: 0, - output_tokens: 0, - cache_read_tokens: 0, - cache_write_tokens: 0, - reasoning_tokens: 0, - total_tokens: 0, - observed_at, - metadata, - created_at: observed_at, - } - } - - fn event_ids(events: &[UsageEvent]) -> Vec<&str> { - events.iter().map(|event| event.event_id.as_str()).collect() + fn event_identity_breaks_remaining_ties() { + assert_eq!( + sorted_ids(vec![ + event(1, timestamp(), "response", "b"), + event(1, timestamp(), "response", "a"), + ]), + ["a", "b"] + ); } } diff --git a/crates/abyss-backend/src/usage/mod.rs b/crates/abyss-backend/src/usage/mod.rs index 0d055c2..a71b0c2 100644 --- a/crates/abyss-backend/src/usage/mod.rs +++ b/crates/abyss-backend/src/usage/mod.rs @@ -7,18 +7,21 @@ /// Image attachment contracts and validation. pub mod attachments; +mod conversation; /// Opaque diagnostic-capture ingest contracts. pub mod diagnostics; pub mod event_order; pub mod persistence; +/// Owner-scoped session index contracts. +pub mod session_list; +pub mod session_title; +pub mod trajectory; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use uuid::Uuid; -const WORKING_DIRECTORY_METADATA_KEY: &str = "working_directory"; - #[derive(Deserialize)] /// Atomic batch accepted by the event-ingestion endpoint. pub struct IngestEventsRequest { @@ -42,17 +45,29 @@ pub struct IngestUsageEvent { pub agent: AgentPayload, /// Agent-native session identifier. pub session_id: String, - /// Collector-side best-effort turn number. Ingest normalizes this when a - /// stable logical-turn/provider identity shows that a restarted collector - /// reused an earlier local sequence number. - pub turn_index: i32, + /// The user turn this request works for, unique within its session; + /// absent when the request works for no particular turn. + pub turn_id: Option, + /// Whether the request comes from a thread the Agent opened only to name + /// another session. + #[serde(default)] + pub naming_thread: bool, + /// The thread that sent this exchange's request, as the Agent declares + /// it; requests with the same value continue one conversation. + pub stream_id: Option, + /// Provider identity of the response this exchange produced; both events + /// of the exchange carry it. + pub response_id: Option, + /// The response this exchange's request continues after; the request then + /// carries only the messages added since. + pub previous_response_id: Option, /// LLM provider and model labels. pub llm: LlmPayload, /// Whether the event represents the request or response side of a call. pub event_type: UsageEventType, /// Optional prompt or response text. pub text: Option, - /// Provider-reported token counters; missing counters default to zero. + /// Token counters of the `AgentEvent`; absent counters are stored as zero. #[serde(default)] pub token_usage: TokenUsagePayload, /// Collector metadata retained with the raw event. @@ -66,10 +81,9 @@ pub struct IngestUsageEvent { #[derive(Debug, Deserialize)] /// Collector-supplied host identity used to group events by device context. pub struct DevicePayload { - /// User-visible host name; `hostname` remains an accepted JSON alias. - #[serde(alias = "hostname")] + /// User-visible host name, stored exactly as received. pub host_name: String, - /// Normalized operating-system or runtime platform label. + /// Operating-system or runtime platform label, stored exactly as received. pub platform: String, /// Optional operating-system version observed by the collector. pub os_version: Option, @@ -78,7 +92,7 @@ pub struct DevicePayload { #[derive(Debug, Deserialize)] /// Collector-supplied Agent identity. pub struct AgentPayload { - /// Agent name, normalized to a canonical slug during ingest. + /// Agent name identified by the collector, stored exactly as received. pub name: String, /// Optional Agent build or release version. pub version: Option, @@ -87,7 +101,7 @@ pub struct AgentPayload { #[derive(Debug, Deserialize)] /// Collector-supplied LLM target identity. pub struct LlmPayload { - /// Provider name, normalized to a lowercase slug during ingest. + /// Provider name, stored exactly as received. pub provider: String, /// Provider-specific model identifier. pub model: String, @@ -118,18 +132,22 @@ impl UsageEventType { clippy::struct_field_names, reason = "Public JSON token usage fields intentionally include the token suffix." )] +/// Token counters of the `AgentEvent`, stored as sent. +/// +/// Input, cache read, cache write, and output counters are disjoint; +/// reasoning is a breakdown of output. pub struct TokenUsagePayload { - /// Input tokens reported by the provider. + /// Input tokens neither read from nor written into a provider cache. pub input_tokens: Option, - /// Output tokens reported by the provider. + /// Output tokens, including reasoning. pub output_tokens: Option, - /// Tokens served from a provider cache. + /// Input tokens served from a provider cache. pub cache_read_tokens: Option, - /// Tokens written to a provider cache. + /// Input tokens written to a provider cache. pub cache_write_tokens: Option, - /// Provider-reported reasoning tokens. + /// Part of output tokens spent on reasoning. pub reasoning_tokens: Option, - /// Provider total, or a derived sum when omitted. + /// Sum of the four disjoint counters. pub total_tokens: Option, } @@ -163,14 +181,18 @@ pub struct SummaryQuery { pub fields: Option, /// User UUID, name, or email filter retained for the summary contract. pub user_filter: Option, - /// Agent-name filter. - pub agent_name: Option, - /// Agent-native session identifier filter. - pub session_id: Option, - /// LLM provider filter. - pub llm_provider: Option, - /// LLM model filter. - pub llm_model: Option, + /// Agent-name filter; repeat the key to match any of several Agents. + #[serde(default)] + pub agent_name: Vec, + /// Agent-native session identifier filter; repeat the key to match any of several. + #[serde(default)] + pub session_id: Vec, + /// LLM provider filter; repeat the key to match any of several providers. + #[serde(default)] + pub llm_provider: Vec, + /// LLM model filter; repeat the key to match any of several models. + #[serde(default)] + pub llm_model: Vec, /// Event-side filter. pub event_type: Option, /// Requested aggregate row limit. @@ -183,7 +205,7 @@ pub struct SummaryQuery { pub enum SummaryFields { /// Return dimensions, event counts, and every token counter. Full, - /// Return only day, input-token, and total-token fields. + /// Return only the day and disjoint token counters used by charts. TokenUsage, } @@ -226,7 +248,7 @@ pub struct SummaryRow { /// Device OS version when grouped by device. #[serde(skip_serializing_if = "Option::is_none")] pub os_version: Option, - /// Canonical Agent name when grouped by Agent. + /// Agent name as reported by the collector, when grouped by Agent. #[serde(skip_serializing_if = "Option::is_none")] pub agent_name: Option, /// LLM provider when grouped by provider. @@ -298,9 +320,15 @@ pub struct TokenUsageSummaryRow { /// UTC calendar day when day grouping was requested. #[serde(skip_serializing_if = "Option::is_none")] pub day: Option, - /// Summed input tokens. + /// Summed input tokens that were neither read from nor written to a cache. pub input_tokens: i64, - /// Summed provider or derived total tokens. + /// Summed output tokens, including reasoning. + pub output_tokens: i64, + /// Summed input tokens read from a provider cache. + pub cache_read_tokens: i64, + /// Summed input tokens written into a provider cache. + pub cache_write_tokens: i64, + /// Summed total of the four disjoint counters. pub total_tokens: i64, } @@ -309,6 +337,9 @@ impl From for TokenUsageSummaryRow { Self { day: row.day, input_tokens: row.input_tokens, + output_tokens: row.output_tokens, + cache_read_tokens: row.cache_read_tokens, + cache_write_tokens: row.cache_write_tokens, total_tokens: row.total_tokens, } } @@ -321,24 +352,49 @@ pub struct RawEventsQuery { pub from: Option>, /// Exclusive upper observation-time bound. pub to: Option>, - /// Agent-name filter. - pub agent_name: Option, - /// Agent-native session identifier filter. - pub session_id: Option, + /// Agent-name filter; repeat the key to match any of several Agents. + #[serde(default)] + pub agent_name: Vec, + /// Agent-native session identifier filter; repeat the key to match any of several. + #[serde(default)] + pub session_id: Vec, /// Backend session primary-key filter. pub session_pk: Option, /// Normalized turn-number filter. pub turn_index: Option, - /// LLM provider filter. - pub llm_provider: Option, - /// LLM model filter. - pub llm_model: Option, + /// LLM provider filter; repeat the key to match any of several providers. + #[serde(default)] + pub llm_provider: Vec, + /// LLM model filter; repeat the key to match any of several models. + #[serde(default)] + pub llm_model: Vec, /// Request or response filter. pub event_type: Option, /// Requested page size. pub limit: Option, /// Number of newest matching events to skip. pub offset: Option, + /// Selects complete events or the content-free list projection. + pub fields: Option, +} + +impl RawEventsQuery { + /// Whether events carry their text, metadata, and attachments. + pub fn includes_content(&self) -> bool { + self.fields.unwrap_or_default() == RawEventFields::Full + } +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +/// Response projection requested from the raw-event endpoint. +pub enum RawEventFields { + /// Return every event field, including text, metadata, and attachments. + #[default] + Full, + /// Return what an event list shows. Text and metadata hold whole request + /// bodies, so storage does not read them and attachments are not loaded. + List, } #[derive(Debug, Serialize)] @@ -350,6 +406,110 @@ pub struct RawEventsResponse { pub next_page_token: Option, } +#[derive(Debug, Serialize)] +/// One page of usage events without their content, for event lists. +pub struct RawEventListResponse { + /// Events ordered from newest to oldest observation time. + pub events: Vec, + /// Decimal offset for the next page, or `None` at the end. + pub next_page_token: Option, +} + +impl RawEventListResponse { + /// Drops the content fields from a page loaded without them. + pub fn from_events(response: RawEventsResponse) -> Self { + Self { + events: response + .events + .into_iter() + .map(UsageEventListItem::from) + .collect(), + next_page_token: response.next_page_token, + } + } +} + +#[derive(Debug, Serialize)] +/// One usage event as an event list shows it: identity, model, Agent, device, +/// and token counts, without text, metadata, or attachments. +pub struct UsageEventListItem { + /// Backend-generated event primary key. + pub id: Uuid, + /// Collector-generated idempotency key. + pub event_id: String, + /// Authenticated owner identifier. + pub user_id: Uuid, + /// Backend device-context primary key. + pub device_context_id: Uuid, + /// Device host name, when the referenced device still exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub host_name: Option, + /// Device platform, when the referenced device still exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, + /// Backend session primary key. + pub session_pk: Uuid, + /// Backend turn primary key. + pub turn_pk: Uuid, + /// Agent name as reported by the collector. + pub agent_name: String, + /// Agent version observed on this event. + pub agent_version: Option, + /// Agent-native session identifier. + pub session_id: String, + /// Backend-normalized turn number. + pub turn_index: i32, + /// LLM provider as reported by the collector. + pub llm_provider: String, + /// Provider-specific model identifier. + pub llm_model: String, + /// Canonical request or response value. + pub event_type: String, + /// Normalized input-token count. + pub input_tokens: i64, + /// Normalized output-token count. + pub output_tokens: i64, + /// Normalized cache-read-token count. + pub cache_read_tokens: i64, + /// Normalized cache-write-token count. + pub cache_write_tokens: i64, + /// Normalized reasoning-token count. + pub reasoning_tokens: i64, + /// Provider-reported or derived total-token count. + pub total_tokens: i64, + /// Collector observation timestamp. + pub observed_at: DateTime, +} + +impl From for UsageEventListItem { + fn from(event: UsageEventResponse) -> Self { + Self { + id: event.id, + event_id: event.event_id, + user_id: event.user_id, + device_context_id: event.device_context_id, + host_name: event.host_name, + platform: event.platform, + session_pk: event.session_pk, + turn_pk: event.turn_pk, + agent_name: event.agent_name, + agent_version: event.agent_version, + session_id: event.session_id, + turn_index: event.turn_index, + llm_provider: event.llm_provider, + llm_model: event.llm_model, + event_type: event.event_type, + input_tokens: event.input_tokens, + output_tokens: event.output_tokens, + cache_read_tokens: event.cache_read_tokens, + cache_write_tokens: event.cache_write_tokens, + reasoning_tokens: event.reasoning_tokens, + total_tokens: event.total_tokens, + observed_at: event.observed_at, + } + } +} + #[derive(Debug, Serialize)] /// API representation of one persisted usage event. pub struct UsageEventResponse { @@ -371,7 +531,7 @@ pub struct UsageEventResponse { pub session_pk: Uuid, /// Backend turn primary key. pub turn_pk: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Agent version observed on this event. pub agent_version: Option, @@ -379,7 +539,7 @@ pub struct UsageEventResponse { pub session_id: String, /// Backend-normalized turn number. pub turn_index: i32, - /// Canonical LLM provider slug. + /// LLM provider as reported by the collector. pub llm_provider: String, /// Provider-specific model identifier. pub llm_model: String, @@ -403,6 +563,15 @@ pub struct UsageEventResponse { pub total_tokens: i64, /// Collector observation timestamp. pub observed_at: DateTime, + /// Agent-declared thread of the exchange. + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_id: Option, + /// Provider identity of the response the exchange produced. + #[serde(skip_serializing_if = "Option::is_none")] + pub response_id: Option, + /// Response whose conversation the exchange's request continues. + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, /// Collector metadata retained without schema expansion. pub metadata: Value, /// Ordered attachment metadata associated with the event. @@ -411,11 +580,63 @@ pub struct UsageEventResponse { #[derive(Debug, Serialize)] /// Session metadata and its ordered turn timelines. -pub struct SessionTimelineResponse { +/// +/// `E` is the per-event representation: complete events for the full +/// timeline, or [`UsageEventOutline`] for the content-free outline. +pub struct SessionTimelineResponse { /// Authoritative session information. pub session: SessionInfo, /// Turns ordered by normalized turn index. - pub turns: Vec, + pub turns: Vec>, +} + +/// Session timeline whose events omit text and metadata bodies. +pub type SessionOutlineResponse = SessionTimelineResponse; + +/// Maximum number of characters returned in an outline event preview. +pub const OUTLINE_PREVIEW_CHARS: i64 = 200; + +#[derive(Debug, Serialize)] +/// Content-free representation of one event, sized independently of its text. +/// +/// Full content is loaded separately by event primary key. +pub struct UsageEventOutline { + /// Backend-generated event primary key. + pub id: Uuid, + /// Collector-generated idempotency key. + pub event_id: String, + /// Owning turn, used only to group events. + #[serde(skip)] + pub turn_pk: Uuid, + /// Backend-normalized turn number. + pub turn_index: i32, + /// LLM provider as reported by the collector. + pub llm_provider: String, + /// Provider-specific model identifier. + pub llm_model: String, + /// Canonical request or response value. + pub event_type: String, + /// Normalized input-token count. + pub input_tokens: i64, + /// Normalized output-token count. + pub output_tokens: i64, + /// Normalized cache-read-token count. + pub cache_read_tokens: i64, + /// Normalized cache-write-token count. + pub cache_write_tokens: i64, + /// Normalized reasoning-token count. + pub reasoning_tokens: i64, + /// Provider-reported or derived total-token count. + pub total_tokens: i64, + /// Collector observation timestamp. + pub observed_at: DateTime, + /// Text length in characters; zero when text is absent. + pub text_chars: i64, + /// Up to [`OUTLINE_PREVIEW_CHARS`] characters of text: the tail of a + /// request, where the newest prompt is, or the head of a response. + pub preview: Option, + /// Number of image attachments on the event. + pub attachment_count: i64, } #[derive(Debug, Serialize)] @@ -427,7 +648,7 @@ pub struct SessionInfo { pub user_id: Uuid, /// Most recently associated device context. pub device_context_id: Uuid, - /// Canonical Agent name. + /// Agent name as reported by the collector. pub agent_name: String, /// Most recently known Agent version. pub agent_version: Option, @@ -439,30 +660,18 @@ pub struct SessionInfo { pub ended_at: Option>, /// Stable, allowlisted session metadata. pub metadata: Value, -} - -/// Extracts the endpoint working directory retained in session metadata. -#[must_use] -pub fn session_working_directory(metadata: &Value) -> Option { - metadata - .get(WORKING_DIRECTORY_METADATA_KEY) - .and_then(Value::as_str) - .filter(|value| !value.is_empty() && value.len() <= 4_096) - .map(str::to_owned) -} - -/// Keeps only stable session-level metadata from an event. -#[must_use] -pub fn session_metadata_from_event(metadata: &Value) -> Value { - session_working_directory(metadata).map_or_else( - || json!({}), - |working_directory| json!({WORKING_DIRECTORY_METADATA_KEY: working_directory}), - ) + /// Agent-generated title, or a first-user-input preview for other Agents. + pub title: Option, + /// Why the session exists when it only serves the Agent, such as + /// `title_generation`; `None` for ordinary sessions. + pub purpose: Option, + /// For a naming thread, the task session its title was matched to. + pub parent_session_pk: Option, } #[derive(Debug, Serialize)] /// One normalized turn and its deterministically ordered events. -pub struct TurnTimeline { +pub struct TurnTimeline { /// Backend turn primary key. pub turn_pk: Uuid, /// Backend-normalized turn number. @@ -472,7 +681,7 @@ pub struct TurnTimeline { /// Latest event observation time in the turn. pub ended_at: Option>, /// Request and response events in authoritative timeline order. - pub events: Vec, + pub events: Vec, } /// Returns the JSON object used when optional metadata is absent. @@ -483,9 +692,8 @@ pub fn empty_metadata() -> Value { #[cfg(test)] mod tests { use super::{ - IngestEventsRequest, SummaryFields, SummaryQuery, SummaryResponse, SummaryRow, - TokenUsageSummaryResponse, UsageEventType, session_metadata_from_event, - session_working_directory, + IngestEventsRequest, RawEventFields, RawEventsQuery, SummaryFields, SummaryQuery, + SummaryResponse, SummaryRow, TokenUsageSummaryResponse, UsageEventType, }; use serde_json::json; @@ -513,10 +721,7 @@ mod tests { }, "event_type": "request", "text": "Explain this repository.", - "token_usage": { - "input_tokens": 10_i64, - "cache_read_tokens": 2_i64 - }, + "token_usage": {"input_tokens": 10_i64, "output_tokens": 0_i64, "cache_read_tokens": 2_i64, "cache_write_tokens": 0_i64, "reasoning_tokens": 0_i64, "total_tokens": 12_i64}, "metadata": { "source": "unit-test" } @@ -559,14 +764,13 @@ mod tests { "event_id": "evt-usage-002", "observed_at": "2026-05-20T01:03:03Z", "device": { - "hostname": "alice-mbp", + "host_name": "alice-mbp", "platform": "macos" }, "agent": { - "name": "Claude Code" + "name": "claude-code" }, "session_id": "session-002", - "turn_index": 2_i32, "llm": { "provider": "Anthropic", "model": "claude-4" @@ -588,7 +792,12 @@ mod tests { ); assert_eq!( event.token_usage.total_tokens, None, - "omitted token_usage should default to empty token counters" + "omitted token_usage should leave every counter absent" + ); + assert_eq!(event.turn_id, None, "an omitted turn names no turn"); + assert!( + !event.naming_thread, + "an omitted flag is not a naming thread" ); assert_eq!( event.metadata, @@ -597,23 +806,6 @@ mod tests { ); } - #[test] - fn session_metadata_keeps_only_a_bounded_working_directory() { - let metadata = json!({ - "working_directory": "/Users/alice/repo", - "request_hash": "sensitive-event-detail" - }); - - assert_eq!( - session_working_directory(&metadata).as_deref(), - Some("/Users/alice/repo") - ); - assert_eq!( - session_metadata_from_event(&metadata), - json!({"working_directory": "/Users/alice/repo"}) - ); - } - #[test] fn summary_query_accepts_token_usage_fields_mode() { let query = serde_json::from_value::(json!({ @@ -624,6 +816,18 @@ mod tests { assert_eq!(query.fields, Some(SummaryFields::TokenUsage)); } + #[test] + fn raw_events_query_selects_the_list_projection() { + let list = serde_json::from_value::(json!({"fields": "list"})) + .expect("list query should deserialize"); + let full = serde_json::from_value::(json!({})) + .expect("default query should deserialize"); + + assert_eq!(list.fields, Some(RawEventFields::List)); + assert!(!list.includes_content()); + assert!(full.includes_content()); + } + #[test] fn token_usage_summary_response_omits_non_token_fields() { let response = TokenUsageSummaryResponse::from_summary(SummaryResponse { @@ -668,6 +872,9 @@ mod tests { { "day": "2026-07-01", "input_tokens": 123_i64, + "output_tokens": 456_i64, + "cache_read_tokens": 11_i64, + "cache_write_tokens": 12_i64, "total_tokens": 615_i64 } ], diff --git a/crates/abyss-backend/src/usage/persistence.rs b/crates/abyss-backend/src/usage/persistence.rs index 1112eae..e8938f8 100644 --- a/crates/abyss-backend/src/usage/persistence.rs +++ b/crates/abyss-backend/src/usage/persistence.rs @@ -9,7 +9,7 @@ use sha2::{Digest, Sha256}; use crate::error::AppError; use super::{ - IngestEventsRequest, IngestUsageEvent, + IngestEventsRequest, attachments::{ValidatedImageAttachment, validate_image_attachments}, }; @@ -27,7 +27,6 @@ pub fn validate_batch( // its table locks are acquired by either storage implementation. let mut attachments = Vec::with_capacity(request.events.len()); for event in &request.events { - validate_event(event)?; attachments.push(validate_image_attachments(&event.attachments)?); } if request.diagnostic_captures.len() > max_batch_size { @@ -52,85 +51,6 @@ pub fn validate_batch( Ok(attachments) } -pub fn validate_event(event: &IngestUsageEvent) -> Result<(), AppError> { - require_non_empty(&event.event_id, "event_id")?; - require_non_empty(&event.device.host_name, "device.host_name")?; - require_non_empty(&event.device.platform, "device.platform")?; - require_non_empty(&event.agent.name, "agent.name")?; - require_non_empty(&event.session_id, "session_id")?; - require_non_empty(&event.llm.provider, "llm.provider")?; - require_non_empty(&event.llm.model, "llm.model")?; - if event.turn_index < 1_i32 { - return Err(AppError::validation("turn_index must be >= 1".to_owned())); - } - let tokens = normalized_tokens(event)?; - if tokens.total == 0 { - return Ok(()); - } - let visible_tokens = tokens - .input - .saturating_add(tokens.output) - .saturating_add(tokens.reasoning); - if tokens.total < visible_tokens { - tracing::debug!( - event_id = %event.event_id, - "provider total_tokens is lower than visible token components" - ); - } - Ok(()) -} - -fn require_non_empty(value: &str, field: &str) -> Result<(), AppError> { - if value.trim().is_empty() { - return Err(AppError::validation(format!("{field} is required"))); - } - Ok(()) -} - -pub struct NormalizedTokens { - pub input: i64, - pub output: i64, - pub cache_read: i64, - pub cache_write: i64, - pub reasoning: i64, - pub total: i64, -} - -pub fn normalized_tokens(event: &IngestUsageEvent) -> Result { - let input = normalize_token(event.token_usage.input_tokens, "input_tokens")?; - let output = normalize_token(event.token_usage.output_tokens, "output_tokens")?; - let cache_read = normalize_token(event.token_usage.cache_read_tokens, "cache_read_tokens")?; - let cache_write = normalize_token(event.token_usage.cache_write_tokens, "cache_write_tokens")?; - let reasoning = normalize_token(event.token_usage.reasoning_tokens, "reasoning_tokens")?; - // Preserve an explicit provider total because provider accounting may not - // equal the visible component sum; derive only when the field is absent. - let total = match event.token_usage.total_tokens { - Some(value) => normalize_token(Some(value), "total_tokens")?, - None => input - .saturating_add(output) - .saturating_add(cache_read) - .saturating_add(cache_write) - .saturating_add(reasoning), - }; - - Ok(NormalizedTokens { - input, - output, - cache_read, - cache_write, - reasoning, - total, - }) -} - -fn normalize_token(value: Option, field: &str) -> Result { - let value = value.unwrap_or(0); - if value < 0 { - return Err(AppError::validation(format!("{field} must be >= 0"))); - } - Ok(value) -} - pub fn validate_event_type(value: &str) -> Result { match value.trim().to_ascii_lowercase().as_str() { "request" => Ok("request".to_owned()), @@ -173,50 +93,28 @@ pub fn parse_group_by(raw: Option<&str>) -> Vec { } } -pub fn normalize_slug(value: &str) -> String { - value - .trim() - .to_ascii_lowercase() - .chars() - .map(|character| match character { - '_' | ' ' => '-', - other => other, - }) - .collect() -} - -pub fn canonical_agent_name(value: &str) -> String { - let normalized = normalize_slug(value); - match normalized.as_str() { - "claude" | "claude-desktop" => "claude-code".to_owned(), - _ => normalized, - } +/// Trimmed values compared exactly, such as Agent names, providers, models, or +/// session identifiers; empty matches every value. +pub fn exact_values_filter(values: &[String]) -> Vec { + distinct( + values + .iter() + .filter_map(|value| non_empty(value)) + .map(normalize_text), + ) } -pub fn agent_name_filter_values(value: &str) -> Vec { - let canonical = canonical_agent_name(value); - if canonical == "claude-code" { - vec![ - "claude-code".to_owned(), - "claude".to_owned(), - "claude-desktop".to_owned(), - ] - } else { - vec![canonical] - } +fn distinct(values: impl Iterator) -> Vec { + let mut values: Vec = values.collect(); + values.sort_unstable(); + values.dedup(); + values } pub fn normalize_text(value: &str) -> String { value.trim().to_owned() } -pub fn normalized_optional(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - pub fn non_empty(value: &str) -> Option<&str> { (!value.trim().is_empty()).then_some(value) } @@ -251,102 +149,24 @@ pub fn sha256_bytes(value: &str) -> Vec { Sha256::digest(value.as_bytes()).to_vec() } -pub struct TurnIdentity { - pub kind: TurnIdentityKind, - pub value: String, -} - -#[derive(Clone, Copy)] -pub enum TurnIdentityKind { - CodexTurnId, - ClaudeTurnId, - ResponseId, - MessageId, - RequestHash, -} - -impl TurnIdentityKind { - pub const fn metadata_key(self) -> &'static str { - match self { - Self::CodexTurnId => "codex_turn_id", - Self::ClaudeTurnId => "claude_turn_id", - Self::ResponseId => "response_id", - Self::MessageId => "message_id", - Self::RequestHash => "request_hash", - } - } -} - -pub fn turn_identity(event: &IngestUsageEvent) -> Option { - turn_identity_from_metadata(&event.metadata) -} - -pub fn turn_identity_from_metadata(metadata: &serde_json::Value) -> Option { - // Prefer Agent-level identities because one user turn may contain several - // provider round trips. - [ - TurnIdentityKind::CodexTurnId, - TurnIdentityKind::ClaudeTurnId, - TurnIdentityKind::ResponseId, - TurnIdentityKind::MessageId, - TurnIdentityKind::RequestHash, - ] - .into_iter() - .find_map(|kind| { - metadata_string(metadata, kind.metadata_key()).map(|value| TurnIdentity { kind, value }) - }) -} - -fn metadata_string(metadata: &serde_json::Value, key: &str) -> Option { - metadata - .get(key) - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -pub const fn choose_turn_index( - requested_turn_index: i32, - existing_turn_index: Option, - next_turn_index: i32, -) -> i32 { - if let Some(existing_turn_index) = existing_turn_index { - return existing_turn_index; - } - if requested_turn_index < next_turn_index { - next_turn_index - } else { - requested_turn_index - } -} - #[cfg(test)] mod tests { - use serde_json::json; - - use super::{ - TurnIdentityKind, canonical_agent_name, choose_turn_index, escape_like_pattern, - parse_group_by, turn_identity_from_metadata, - }; + use super::{escape_like_pattern, exact_values_filter, parse_group_by}; #[test] fn shared_normalization_is_backend_independent() { - assert_eq!(canonical_agent_name(" Claude Desktop "), "claude-code"); + // Agent names and providers are matched exactly as reported: Claude + // Desktop stays apart from Claude Code and case is not folded. + assert_eq!( + exact_values_filter(&[ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "Codex".to_owned(), + "claude-code".to_owned(), + ]), + ["Codex", "claude-code", "claude-desktop"] + ); assert_eq!(parse_group_by(Some("day,llm_model")), ["day", "model"]); assert_eq!(escape_like_pattern(r"a%b_c\d"), r"a\%b\_c\\d"); } - - #[test] - fn turn_identity_and_restart_resolution_are_stable() { - let identity = turn_identity_from_metadata(&json!({ - "response_id": "response", - "codex_turn_id": "turn" - })) - .expect("turn identity should be extracted"); - assert!(matches!(identity.kind, TurnIdentityKind::CodexTurnId)); - assert_eq!(identity.value, "turn"); - assert_eq!(choose_turn_index(1_i32, None, 4_i32), 4_i32); - assert_eq!(choose_turn_index(1_i32, Some(2_i32), 4_i32), 2_i32); - } } diff --git a/crates/abyss-backend/src/usage/session_list.rs b/crates/abyss-backend/src/usage/session_list.rs new file mode 100644 index 0000000..bd931fb --- /dev/null +++ b/crates/abyss-backend/src/usage/session_list.rs @@ -0,0 +1,213 @@ +//! Owner-scoped session index: one page of sessions ordered by latest activity. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{error::AppError, usage::persistence::exact_values_filter}; + +const DEFAULT_PAGE_SIZE: u32 = 20; +const MAX_PAGE_SIZE: u32 = 100; + +/// Query parameters for `GET /v1/agent-usage/sessions`. +#[derive(Debug, Default, Deserialize)] +pub struct SessionListQuery { + /// Keeps sessions whose latest activity is at or after this instant. + pub from: Option>, + /// Keeps sessions that started before this instant. + pub to: Option>, + /// Agent-name filter, matched exactly as reported; repeat the key for + /// several Agents. + #[serde(default)] + pub agent_name: Vec, + /// Keeps sessions with at least one event from any of these LLM providers. + #[serde(default)] + pub llm_provider: Vec, + /// Keeps sessions with at least one event from any of these LLM models. + #[serde(default)] + pub llm_model: Vec, + /// 1-based page number. + pub page: Option, + /// Sessions per page, at most 100. + pub page_size: Option, +} + +/// Session list query with normalized filters and a resolved page window. +#[derive(Debug)] +pub struct ValidatedSessionListQuery { + pub from: Option>, + pub to: Option>, + /// Stored agent names to match, or empty for every Agent. + pub agent_names: Vec, + /// Exact providers; a session must have used one, or empty for any. + pub llm_providers: Vec, + /// Exact models; a session must have used one, or empty for any. + pub llm_models: Vec, + pub page: u32, + pub page_size: u32, + pub offset: i64, +} + +impl SessionListQuery { + pub fn validate(self) -> Result { + if let (Some(from), Some(to)) = (self.from, self.to) + && from >= to + { + return Err(AppError::validation( + "from must be earlier than to".to_owned(), + )); + } + let page = self.page.unwrap_or(1); + if page == 0 { + return Err(AppError::validation("page must be at least 1".to_owned())); + } + let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); + if page_size == 0 || page_size > MAX_PAGE_SIZE { + return Err(AppError::validation(format!( + "page_size must be between 1 and {MAX_PAGE_SIZE}" + ))); + } + let offset = i64::from(page.saturating_sub(1)).saturating_mul(i64::from(page_size)); + Ok(ValidatedSessionListQuery { + from: self.from, + to: self.to, + agent_names: exact_values_filter(&self.agent_name), + llm_providers: exact_values_filter(&self.llm_provider), + llm_models: exact_values_filter(&self.llm_model), + page, + page_size, + offset, + }) + } +} + +/// One page of sessions, newest activity first. +#[derive(Debug, Serialize)] +pub struct SessionListResponse { + /// Sessions matching the filters across all pages. + pub total_sessions: i64, + pub page: u32, + pub page_size: u32, + pub has_more: bool, + pub items: Vec, +} + +/// Summary of one session for the session index. +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct SessionListItem { + pub session_pk: Uuid, + pub session_id: String, + pub agent_name: String, + pub agent_version: Option, + pub host_name: String, + pub platform: String, + pub started_at: DateTime, + pub ended_at: Option>, + /// Latest event observation, or the session start when it has no events. + pub last_activity_at: DateTime, + pub turns: i64, + pub events: i64, + pub total_tokens: i64, + /// Agent-generated title, or a first-user-input preview for other Agents. + pub title: Option, + /// Why the session exists when it only serves the Agent, such as + /// `title_generation`; `None` for ordinary sessions. + pub purpose: Option, +} + +impl SessionListResponse { + pub fn new( + query: &ValidatedSessionListQuery, + total_sessions: i64, + items: Vec, + ) -> Self { + let shown = query + .offset + .saturating_add(i64::try_from(items.len()).unwrap_or(i64::MAX)); + Self { + total_sessions, + page: query.page, + page_size: query.page_size, + has_more: shown < total_sessions, + items, + } + } +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + + #[test] + fn defaults_to_the_first_page_of_every_agent() { + let query = SessionListQuery::default().validate().expect("valid"); + assert_eq!((query.page, query.page_size, query.offset), (1, 20, 0)); + assert_eq!(query.agent_names, Vec::::new()); + } + + #[test] + fn keeps_agent_names_exact_and_computes_the_offset() { + let query = SessionListQuery { + agent_name: vec![ + " claude-desktop ".to_owned(), + "claude-code".to_owned(), + "codex".to_owned(), + ], + page: Some(3), + page_size: Some(25), + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert_eq!(query.offset, 50); + assert_eq!( + query.agent_names, + ["claude-code", "claude-desktop", "codex"] + ); + } + + #[test] + fn keeps_providers_and_models_verbatim() { + let query = SessionListQuery { + llm_provider: vec![" OpenAI ".to_owned(), "anthropic".to_owned()], + llm_model: vec![" gpt-5 ".to_owned(), "Claude-Opus".to_owned()], + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert_eq!(query.llm_providers, ["OpenAI", "anthropic"]); + assert_eq!(query.llm_models, ["Claude-Opus", "gpt-5"]); + let blank = SessionListQuery { + llm_model: vec![" ".to_owned()], + ..SessionListQuery::default() + } + .validate() + .expect("valid"); + assert_eq!(blank.llm_models, Vec::::new()); + } + + #[test] + fn rejects_bad_windows_and_page_sizes() { + let at = Utc.with_ymd_and_hms(2026, 9, 1, 0, 0, 0).unwrap(); + let reversed = SessionListQuery { + from: Some(at), + to: Some(at), + ..SessionListQuery::default() + }; + assert!(reversed.validate().is_err()); + for page_size in [0, 101] { + let query = SessionListQuery { + page_size: Some(page_size), + ..SessionListQuery::default() + }; + assert!(query.validate().is_err()); + } + let zero_page = SessionListQuery { + page: Some(0), + ..SessionListQuery::default() + }; + assert!(zero_page.validate().is_err()); + } +} diff --git a/crates/abyss-backend/src/usage/session_title.rs b/crates/abyss-backend/src/usage/session_title.rs new file mode 100644 index 0000000..733062f --- /dev/null +++ b/crates/abyss-backend/src/usage/session_title.rs @@ -0,0 +1,190 @@ +//! Session names: the title an Agent generated for its own session, else the +//! session's first user input. +//! +//! The collector reports a naming call as event fields: both sides carry a +//! `purpose`, and the response carries the `session_title` it produced for its +//! own session. Titles are therefore read without interpreting message text. +//! +//! An Agent may instead name a task from a separate thread whose requests do +//! not say which session they name. The collector marks those requests with +//! `naming_thread`; their sessions are only marked, and the task sessions keep +//! their first user input. + +use serde_json::Value; + +use super::conversation::Conversation; + +/// Session purpose stored for threads that name another session. +pub const TITLE_GENERATION_PURPOSE: &str = "title_generation"; +/// Longest default title, in characters. +const DEFAULT_TITLE_CHARS: usize = 80; + +/// What one stored event contributes to its session's name. +#[derive(Debug, PartialEq, Eq)] +pub enum TitleContribution<'a> { + /// Title the Agent generated for this event's session. + Agent(&'a str), + /// An ordinary request; its first user input may name the session. + UserInput, + /// The session is a thread that names another session. + TitleThread, +} + +impl<'a> TitleContribution<'a> { + /// Classifies an event from collector fields without reading its messages. + /// Requests the Agent made for its own purposes, such as naming calls, are + /// not user input. + pub fn of(event_type: &str, metadata: &'a Value, naming_thread: bool) -> Option { + if let Some(title) = metadata.get("session_title").and_then(Value::as_str) { + return Some(Self::Agent(title)); + } + if event_type != "request" || metadata.get("purpose").is_some() { + return None; + } + if naming_thread { + return Some(Self::TitleThread); + } + Some(Self::UserInput) + } +} + +/// Default name: the first non-empty user input, on one line, at most 80 chars. +pub fn default_title(metadata: &Value) -> Option { + let input = Conversation::read(metadata)? + .messages + .into_iter() + .find(|message| message.role == "user" && !message.text.trim().is_empty())? + .text; + let input = input.split_whitespace().collect::>().join(" "); + let mut chars = input.chars(); + let mut title: String = chars.by_ref().take(DEFAULT_TITLE_CHARS).collect(); + if chars.next().is_some() { + title.pop(); + title.push('…'); + } + (!title.is_empty()).then_some(title) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + /// Metadata from the current collector, which always reports completeness. + fn request(conversation: &Value) -> Value { + let mut conversation = conversation.clone(); + if conversation.get("completeness").is_none() { + conversation["completeness"] = json!("complete"); + } + json!({"conversation": conversation}) + } + + fn default_of(messages: &Value) -> Option { + default_title(&request(&json!({"messages": messages}))) + } + + #[test] + fn default_title_reads_the_first_nonempty_user_input() { + assert_eq!( + default_of(&json!([ + {"role":"system","text":"Instructions"}, + {"role":"assistant","text":"Previous reply"}, + {"role":"tool","text":"Tool output"}, + {"role":"user","text":" "}, + {"role":"user","text":" 修复\n 登录问题 "}, + {"role":"user","text":"Second request"} + ])) + .as_deref(), + Some("修复 登录问题") + ); + } + + #[test] + fn default_title_truncates_unicode_and_leaves_short_input_intact() { + for (input, expected) in [ + ("修复登录".to_owned(), "修复登录".to_owned()), + ("修".repeat(80), "修".repeat(80)), + ("修".repeat(81), format!("{}…", "修".repeat(79))), + ] { + assert_eq!( + default_of(&json!([{"role":"user","text":input}])), + Some(expected) + ); + } + } + + #[test] + fn default_title_requires_readable_user_input() { + for messages in [ + json!([]), + json!([{"role":"system","text":"Instructions"}]), + json!([{"role":"user","text":" \n "}]), + json!([{"role":"tool","text":"Output"}]), + ] { + assert_eq!(default_of(&messages), None); + } + assert_eq!(default_title(&json!({"text":"No role"})), None); + } + + #[test] + fn agent_titles_come_only_from_the_collector_field() { + assert_eq!( + TitleContribution::of( + "response", + &json!({"purpose":"session_title","session_title":"登录修复"}), + false + ), + Some(TitleContribution::Agent("登录修复")) + ); + // A reply that merely looks like a title object is ordinary output. + assert_eq!( + TitleContribution::of( + "response", + &request(&json!({"messages":[ + {"role":"assistant","text":"{\"title\":\"Quoted\"}"} + ]})), + false + ), + None + ); + } + + #[test] + fn naming_requests_are_not_user_input() { + let mut metadata = request(&json!({"messages":[{"role":"user","text":"…"}]})); + metadata["purpose"] = json!("session_title"); + assert_eq!(TitleContribution::of("request", &metadata, false), None); + } + + #[test] + fn naming_threads_are_marked_from_the_collector_flag() { + let metadata = + request(&json!({"messages":[{"role":"user","text":"Generate a concise title"}]})); + assert_eq!( + TitleContribution::of("request", &metadata, true), + Some(TitleContribution::TitleThread) + ); + assert_eq!( + TitleContribution::of("request", &metadata, false), + Some(TitleContribution::UserInput) + ); + } + + #[test] + fn incomplete_conversations_give_no_default() { + for status in ["partial", "unsupported"] { + let metadata = request( + &json!({"completeness":status,"messages":[{"role":"user","text":"hello"}]}), + ); + assert_eq!(default_title(&metadata), None); + } + } + + #[test] + fn content_outside_collector_messages_is_never_interpreted() { + let metadata = json!({"raw_body":{"parts":[{"content_base64":"e30="}]}, + "conversation":{"messages":[], "completeness":"complete", "provider":{"input":"secret"}}}); + assert_eq!(default_title(&metadata), None); + } +} diff --git a/crates/abyss-backend/src/usage/trajectory.rs b/crates/abyss-backend/src/usage/trajectory.rs new file mode 100644 index 0000000..9329c9a --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory.rs @@ -0,0 +1,1251 @@ +//! Merged session trajectory built from accumulated request/response snapshots. +//! +//! Collectors record each provider request verbatim, so every request repeats +//! the conversation so far. The trajectory keeps only what each event adds: +//! role-aware messages, first-seen tool results, and response text and tool +//! calls, with each tool call paired to its result by call identifier. + +mod conversation; + +use conversation::{ConversationProjection, Placement}; + +use std::{ + collections::{HashMap, HashSet, hash_map::DefaultHasher}, + hash::{Hash, Hasher}, +}; + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde_json::Value; +use uuid::Uuid; + +use super::{SessionInfo, SessionTimelineResponse, UsageEventResponse}; + +/// Purpose the collector declares on both events of a call that asks the +/// model to summarize the conversation so far. +const COMPACTION_PURPOSE: &str = "compaction"; + +#[derive(Debug, Serialize)] +/// Session information and its merged, de-duplicated trajectory items. +pub struct SessionTrajectoryResponse { + /// Authoritative session information. + pub session: SessionInfo, + /// Items in authoritative event order. + pub items: Vec, + /// Every observed request and response, including events with no new content. + pub events: Vec, +} + +#[derive(Debug, Serialize)] +/// Lightweight usage summary independent of content de-duplication. +pub struct TrajectoryEvent { + /// Event that owns these counters. + pub event_pk: Uuid, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Canonical request or response value. + pub event_type: String, + /// Observation time of the event. + pub observed_at: DateTime, + /// Model requested or returned by the provider. + pub llm_model: String, + /// Uncached input tokens; cache counters are separate. + pub input_tokens: i64, + /// Output tokens, including reasoning. + pub output_tokens: i64, + /// Input tokens read from a provider cache. + pub cache_read_tokens: i64, + /// Input tokens written into a provider cache. + pub cache_write_tokens: i64, + /// Reasoning breakdown of output tokens. + pub reasoning_tokens: i64, + /// Authoritative total for this event, counted once. + pub total_tokens: i64, + /// Whether any token counter is non-zero. + pub token_usage_available: bool, +} + +impl From<&UsageEventResponse> for TrajectoryEvent { + fn from(event: &UsageEventResponse) -> Self { + let has_usage = event.input_tokens != 0 + || event.output_tokens != 0 + || event.cache_read_tokens != 0 + || event.cache_write_tokens != 0 + || event.reasoning_tokens != 0 + || event.total_tokens != 0; + Self { + event_pk: event.id, + turn_index: event.turn_index, + event_type: event.event_type.clone(), + observed_at: event.observed_at, + llm_model: event.llm_model.clone(), + input_tokens: event.input_tokens, + output_tokens: event.output_tokens, + cache_read_tokens: event.cache_read_tokens, + cache_write_tokens: event.cache_write_tokens, + reasoning_tokens: event.reasoning_tokens, + total_tokens: event.total_tokens, + token_usage_available: has_usage, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +/// Kind of one trajectory item. +pub enum TrajectoryItemKind { + /// A conversation message not seen earlier in its stream, with the role + /// the collector reported. + Message, + /// Event text of an event whose conversation could not be read. + Text, + /// Response tool call, with its result once one arrives. + ToolCall, + /// Request tool result whose call was not observed. + ToolResult, + /// Context compaction: the new user input of the compaction request as + /// `input` and the summary its response returned as `output`. + Compaction, +} + +#[derive(Debug, Serialize)] +/// One piece of new content contributed by one event. +pub struct TrajectoryItem { + /// Item kind. + pub kind: TrajectoryItemKind, + /// Role of a message item, exactly as the collector reported it. + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Zero-based stream, placed by the identities the collector declares. + pub stream: usize, + /// Backend-normalized turn number. + pub turn_index: i32, + /// Event that contributed this item. + pub event_pk: Uuid, + /// Observation time of that event. + pub observed_at: DateTime, + /// Model of that event. + pub llm_model: String, + /// Text of message and text items. + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// Whether `text` is only the new tail of a paragraph that grew. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub continued: bool, + /// Tool name of a tool call. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Provider call identifier of a tool call or result. + #[serde(skip_serializing_if = "Option::is_none")] + pub call_id: Option, + /// Tool call input. + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + /// Tool result output. + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + /// Event that first carried the tool result. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_event_pk: Option, +} + +impl TrajectoryItem { + fn new(kind: TrajectoryItemKind, stream: usize, event: &UsageEventResponse) -> Self { + Self { + kind, + role: None, + stream, + turn_index: event.turn_index, + event_pk: event.id, + observed_at: event.observed_at, + llm_model: event.llm_model.clone(), + text: None, + continued: false, + name: None, + call_id: None, + input: None, + output: None, + output_event_pk: None, + } + } +} + +/// Merges an ordered session timeline into its trajectory. +pub fn build(timeline: SessionTimelineResponse) -> SessionTrajectoryResponse { + let mut merger = Merger::default(); + let mut events = Vec::new(); + for turn in &timeline.turns { + for event in &turn.events { + events.push(TrajectoryEvent::from(event)); + if event.event_type == "request" { + merger.request(event); + } else { + merger.response(event); + } + } + } + SessionTrajectoryResponse { + session: timeline.session, + items: merger.items, + events, + } +} + +#[derive(Default)] +struct Merger { + conversation: ConversationProjection, + items: Vec, + seen_results: HashSet, + /// Tool call item index by call identifier. + calls: HashMap, + /// Compaction item index by the response id of its exchange, until the + /// response carrying the summary arrives. + compactions: HashMap, +} + +impl Merger { + fn request(&mut self, event: &UsageEventResponse) { + let first_new = self.items.len(); + let placement = self.conversation.place(event); + if let Some(items) = self.conversation.project(event, placement) { + self.items.extend(items); + } else { + self.unclassified_request(event, placement); + } + if is_compaction(event) { + self.compaction(event, placement.stream, first_new); + } + self.request_tools(event, placement.stream); + } + + fn unclassified_request(&mut self, event: &UsageEventResponse, placement: Placement) { + if let Some(text) = event.text.as_ref().filter(|text| !text.is_empty()) { + let mut item = TrajectoryItem::new(TrajectoryItemKind::Text, placement.stream, event); + item.text = Some(text.clone()); + self.items.push(item); + } + } + + fn request_tools(&mut self, event: &UsageEventResponse, stream: usize) { + for segment in content_segments(&event.metadata) { + if segment_str(segment, "type") != Some("tool_result") { + continue; + } + let call_id = segment_str(segment, "call_id").map(str::to_owned); + let output = segment_str(segment, "output").map(str::to_owned); + let result_key = call_id.clone().unwrap_or_else(|| { + format!( + "output:{:x}", + hash_str(output.as_deref().unwrap_or_default()) + ) + }); + if !self.seen_results.insert(result_key) { + continue; + } + if let Some(&index) = call_id.as_ref().and_then(|id| self.calls.get(id)) { + let call = &mut self.items[index]; + call.output = output; + call.output_event_pk = Some(event.id); + } else { + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolResult, stream, event); + item.call_id = call_id; + item.output = output; + item.output_event_pk = Some(event.id); + self.items.push(item); + } + } + } + + /// Merges the new user input of a compaction request into one compaction + /// item, which the response of the same exchange completes with its + /// summary. + fn compaction(&mut self, event: &UsageEventResponse, stream: usize, first_new: usize) { + let mut compaction = TrajectoryItem::new(TrajectoryItemKind::Compaction, stream, event); + let mut index = None; + let mut input = Vec::new(); + for item in self.items.split_off(first_new) { + if item.role.as_deref() != Some("user") { + self.items.push(item); + continue; + } + index.get_or_insert(self.items.len()); + input.extend(item.text); + } + compaction.input = (!input.is_empty()).then(|| input.join("\n\n")); + let index = index.unwrap_or(self.items.len()); + self.items.insert(index, compaction); + if let Some(id) = &event.response_id { + self.compactions.insert(id.clone(), index); + } + } + + fn response(&mut self, event: &UsageEventResponse) { + let placement = self.conversation.place(event); + let stream = placement.stream; + let projected = self.conversation.project(event, placement); + let compaction = event + .response_id + .as_ref() + .and_then(|id| self.compactions.remove(id)); + if let Some(index) = compaction { + // The response of a compaction exchange is the summary. + if let Some(summary) = event.text.as_deref().map(str::trim) + && !summary.is_empty() + && let Some(item) = self.items.get_mut(index) + { + item.output = Some(summary.to_owned()); + item.output_event_pk = Some(event.id); + } + } else if let Some(items) = projected { + self.items.extend(items); + } else if let Some(text) = event.text.as_deref().filter(|text| !text.trim().is_empty()) { + let mut item = TrajectoryItem::new(TrajectoryItemKind::Text, stream, event); + item.text = Some(text.to_owned()); + self.items.push(item); + } + for segment in content_segments(&event.metadata) { + if segment_str(segment, "type") != Some("tool_call") { + continue; + } + let mut item = TrajectoryItem::new(TrajectoryItemKind::ToolCall, stream, event); + item.name = segment_str(segment, "name").map(str::to_owned); + item.call_id = segment_str(segment, "call_id").map(str::to_owned); + item.input = segment_str(segment, "input").map(str::to_owned); + if let Some(call_id) = &item.call_id { + self.calls.insert(call_id.clone(), self.items.len()); + } + self.items.push(item); + } + } +} + +fn is_compaction(event: &UsageEventResponse) -> bool { + event.metadata.get("purpose").and_then(Value::as_str) == Some(COMPACTION_PURPOSE) +} + +fn hash_str(value: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() +} + +fn content_segments(metadata: &Value) -> &[Value] { + metadata + .get("content_segments") + .and_then(Value::as_array) + .map_or(&[], Vec::as_slice) +} + +fn segment_str<'a>(segment: &'a Value, key: &str) -> Option<&'a str> { + segment.get(key).and_then(Value::as_str) +} + +#[cfg(test)] +mod tests { + + /// The role of a message item, else the item kind, as the API reports it. + fn shown(item: &TrajectoryItem) -> &str { + item.role.as_deref().unwrap_or(match item.kind { + TrajectoryItemKind::Message => "message", + TrajectoryItemKind::Text => "text", + TrajectoryItemKind::ToolCall => "tool_call", + TrajectoryItemKind::ToolResult => "tool_result", + TrajectoryItemKind::Compaction => "compaction", + }) + } + + #[test] + fn usage_retains_events_without_new_content_and_counts_each_event_once() { + let body = r#"{"messages":[{"id":null,"role":"user","text":"hello"}],"instructions_present":false}"#; + let mut request = structured_event("request", body); + request.input_tokens = 2; + request.cache_read_tokens = 77_000; + request.cache_write_tokens = 400; + request.total_tokens = 77_402; + let repeated = structured_event("request", body); + let mut response = event("response", "", json!({})); + response.llm_model = "other-model".to_owned(); + response.output_tokens = 150; + response.reasoning_tokens = 40; + response.total_tokens = 150; + let expected_ids = vec![request.id, repeated.id, response.id]; + + let trajectory = build(timeline(vec![request, repeated, response])); + assert_eq!(trajectory.items.len(), 1); + assert_eq!( + trajectory + .events + .iter() + .map(|event| event.event_pk) + .collect::>(), + expected_ids + ); + assert_eq!(trajectory.events[0].total_tokens, 77_402); + assert_eq!(trajectory.events[0].cache_read_tokens, 77_000); + assert_eq!(trajectory.events[0].cache_write_tokens, 400); + assert!(trajectory.events[0].token_usage_available); + assert!(!trajectory.events[1].token_usage_available); + assert_eq!(trajectory.events[2].llm_model, "other-model"); + assert_eq!(trajectory.events[2].output_tokens, 150); + assert_eq!(trajectory.events[2].reasoning_tokens, 40); + assert!(trajectory.events[2].token_usage_available); + let serialized = serde_json::to_value(&trajectory.events).unwrap(); + assert!(serialized[0].get("text").is_none()); + assert!(serialized[0].get("metadata").is_none()); + } + + #[test] + fn usage_is_available_when_any_counter_is_non_zero() { + let zero = event("response", "", json!({})); + let mut cached = event("request", "", json!({})); + cached.cache_read_tokens = 10; + let trajectory = build(timeline(vec![zero, cached])); + assert_eq!(trajectory.events.len(), 2); + assert!(!trajectory.events[0].token_usage_available); + assert!(trajectory.events[1].token_usage_available); + let serialized = serde_json::to_value(&trajectory.events).unwrap(); + assert!(serialized[0].get("token_usage_estimated").is_none()); + } + + use serde_json::json; + + use super::*; + use crate::usage::TurnTimeline; + + #[test] + fn unclassified_requests_keep_every_observation_and_pair_tool_results() { + let system = "system: header cch=1\nYou are an agent."; + let events = vec![ + event("request", &format!("{system}\n\nfirst prompt"), json!({})), + event( + "response", + "", + json!({"content_segments": [ + {"type": "tool_call", "call_id": "c1", "name": "Bash", "input": "ls"} + ]}), + ), + event( + "request", + "system: header cch=2\nYou are an agent.\n\nfirst prompt", + json!({"content_segments": [ + {"type": "tool_result", "call_id": "c1", "output": "README.md"} + ]}), + ), + event("response", "done", json!({})), + event( + "request", + "system: header cch=3\nYou are an agent.\n\nfirst prompt\n\nsecond prompt", + json!({"content_segments": [ + {"type": "tool_result", "call_id": "c1", "output": "README.md"} + ]}), + ), + ]; + + let items = build(timeline(events)).items; + + let kinds = items.iter().map(shown).collect::>(); + assert_eq!(kinds, ["text", "tool_call", "text", "text", "text",]); + assert_eq!(items[1].output.as_deref(), Some("README.md")); + assert_eq!( + items[4].text.as_deref(), + Some("system: header cch=3\nYou are an agent.\n\nfirst prompt\n\nsecond prompt") + ); + assert!(items.iter().all(|item| item.stream == 0)); + } + + #[test] + fn unclassified_requests_are_not_split_trimmed_or_grouped_by_text() { + let transcript = "x".repeat(300); + let events = vec![ + event("request", "system: main\nMain agent.\n\nhello", json!({})), + event( + "request", + &format!("system: classifier\nJudge.\n\n{transcript}"), + json!({}), + ), + event( + "request", + &format!("system: classifier\nJudge.\n\n{transcript} more"), + json!({}), + ), + ]; + + let items = build(timeline(events)).items; + + assert_eq!(items.len(), 3); + assert!(items.iter().all(|i| shown(i) == "text" && !i.continued)); + assert_eq!( + items[2].text, + Some(format!( + "system: classifier\nJudge.\n\n{transcript} more" + )) + ); + } + + fn user_request(texts: &[&str]) -> UsageEventResponse { + structured_event("request", &json!({"messages": texts.iter().map(|text| json!({"role":"user", "text":text})).collect::>()}).to_string()) + } + + #[test] + fn unavailable_roles_never_become_system_or_user_by_text_guessing() { + let text = "system: instructions\n\n用户内容\n\n用户内容"; + for metadata in [ + json!({}), + json!({"conversation":{"completeness":"partial","messages":[]}}), + json!({"conversation":{"messages":"invalid"}}), + json!({"conversation":{"messages":[],"completeness":"complete"}}), + // Without completeness the conversation is outside the contract. + json!({"conversation":{"messages":[{"role":"user","text":"用户内容"}]}}), + ] { + let items = build(timeline(vec![event("request", text, metadata)])).items; + assert_eq!(items.len(), 1); + assert_eq!(shown(&items[0]), "text"); + assert_eq!(items[0].text.as_deref(), Some(text)); + assert!(!items[0].continued); + } + let items = build(timeline(vec![structured_event("request", r#"{"messages":[{"id":null,"role":"unknown","text":"system: unknown"},{"id":null,"role":"unrecognized","text":"hello"},{"id":null,"role":"user","text":"system: quoted text"}],"instructions_present":false}"#)])).items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["unknown", "unrecognized", "user"] + ); + } + + /// Every conversation in the contract declares completeness; default it to complete. + fn structured_event(side: &str, body: &str) -> UsageEventResponse { + let mut conversation = serde_json::from_str::(body).unwrap(); + conversation + .as_object_mut() + .unwrap() + .entry("completeness") + .or_insert_with(|| json!("complete")); + event( + side, + "flattened text", + json!({"conversation": conversation}), + ) + } + + #[test] + fn structured_projection_keeps_roles_and_repeated_user_occurrences() { + let first = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true}"#; + let response = r#"{"messages":[{"id":"a1","role":"assistant","text":"working"}],"instructions_present":false}"#; + let replay = r#"{"messages":[{"role":"developer","text":"system instructions"},{"id":null,"role":"user","text":"继续"},{"id":"a1","role":"assistant","text":"working"},{"id":null,"role":"user","text":"继续"}],"instructions_present":true}"#; + let events = vec![ + structured_event("request", first), + structured_event("response", response), + structured_event("request", replay), + structured_event("request", replay), + ]; + let metadata_before = events + .iter() + .map(|e| e.metadata.clone()) + .collect::>(); + let mut merger = Merger::default(); + for e in &events { + if e.event_type == "request" { + merger.request(e); + } else { + merger.response(e); + } + } + assert_eq!( + merger.items.iter().map(shown).collect::>(), + ["developer", "user", "assistant", "user"] + ); + assert_eq!(merger.items[1].text.as_deref(), Some("继续")); + assert_eq!(merger.items[3].text.as_deref(), Some("继续")); + assert_eq!( + metadata_before, + events + .iter() + .map(|e| e.metadata.clone()) + .collect::>() + ); + } + + #[test] + fn streamed_anthropic_response_is_not_replayed_as_user_input() { + let response = r#"{"messages":[{"role":"assistant","text":"hi"}]}"#; + let items = build(timeline(vec![ + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#), + structured_event("response", response), + structured_event("request", r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#), + ])).items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["system", "user", "assistant", "user"] + ); + assert_eq!(items[2].text.as_deref(), Some("hi")); + } + + #[test] + fn anthropic_reasoning_is_shown_once_with_its_assistant_message() { + let first = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true,"completeness":"complete"}"#; + let response = r#"{"messages":[{"id":"msg_1","role":"reasoning","text":"thinking it over"},{"id":"msg_1","role":"assistant","text":"hi"}],"instructions_present":false,"completeness":"complete"}"#; + let replay = r#"{"messages":[{"id":null,"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"reasoning","text":"thinking it over"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"next"}],"instructions_present":true,"completeness":"complete"}"#; + let items = build(timeline(vec![ + structured_event("request", first), + structured_event("response", response), + structured_event("request", replay), + ])) + .items; + assert_eq!( + items + .iter() + .map(|i| (shown(i), i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + ("system", "instructions"), + ("user", "hello"), + ("reasoning", "thinking it over"), + ("assistant", "hi"), + ("user", "next"), + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + + #[test] + fn incremental_requests_keep_identical_new_user_messages() { + let mut first = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"id":null,"role":"user","text":"continue"}]}"#, + ); + first.response_id = Some("r1".to_owned()); + let mut response = structured_event( + "response", + r#"{"messages":[{"id":null,"role":"assistant","text":"working"}]}"#, + ); + response.response_id = Some("r1".to_owned()); + let mut next = structured_event( + "request", + r#"{"messages":[{"id":null,"role":"user","text":"continue"}]}"#, + ); + next.previous_response_id = Some("r1".to_owned()); + let items = build(timeline(vec![first, response, next])).items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["developer", "user", "assistant", "user"] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + + #[test] + fn a_continuation_shows_new_developer_messages_but_not_its_instructions() { + let mut first = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"role":"user","text":"start"}],"instructions_present":true}"#, + ); + first.response_id = Some("r1".to_owned()); + let mut response = structured_event( + "response", + r#"{"messages":[{"role":"assistant","text":"working"}]}"#, + ); + response.response_id = Some("r1".to_owned()); + // The continuation repeats its instructions, then adds a developer + // notice and the next user input. + let mut next = structured_event( + "request", + r#"{"messages":[{"role":"developer","text":"instructions"},{"role":"developer","text":"permissions changed"},{"role":"user","text":"go on"}],"instructions_present":true}"#, + ); + next.previous_response_id = Some("r1".to_owned()); + let items = build(timeline(vec![first, response, next])).items; + assert_eq!( + texts(&items), + [ + ("developer", 0, "instructions"), + ("user", 0, "start"), + ("assistant", 0, "working"), + ("developer", 0, "permissions changed"), + ("user", 0, "go on"), + ] + ); + } + + #[test] + fn structured_request_retains_system_and_assistant_roles() { + let body = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"user","text":"hello"}],"instructions_present":true}"#; + let items = build(timeline(vec![ + structured_event("request", body), + structured_event("request", body), + ])) + .items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["system", "user", "assistant", "user"] + ); + } + + /// Claude Code requests as the collector projects them: a `create` + /// carries the whole system prompt and history, a `continue` only the + /// billing header and the messages added since the response it names. + fn claude_request( + stream: Option<&str>, + previous: Option<&str>, + response: Option<&str>, + system: &[&str], + messages: Value, + ) -> UsageEventResponse { + let system = std::iter::once( + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; cch=3ceb5;", + ) + .chain(system.iter().copied()) + .collect::>() + .join("\n"); + let mut conversation = json!({ + "messages": [{"id": null, "role": "system", "text": system}], + "instructions_present": true, + }); + conversation["messages"] + .as_array_mut() + .unwrap() + .extend(serde_json::from_value::>(messages).unwrap()); + let mut request = structured_event("request", &conversation.to_string()); + request.stream_id = stream.map(str::to_owned); + request.previous_response_id = previous.map(str::to_owned); + request.response_id = response.map(str::to_owned); + request + } + + /// A main-loop request with its whole history; `response` names the + /// response its exchange produced. + fn create(response: &str, messages: Value) -> UsageEventResponse { + claude_request( + Some("main"), + None, + Some(response), + &["You are Claude Code.", "Main loop instructions."], + messages, + ) + } + + /// A main-loop request continuing after `previous`. + fn continue_from( + previous: &str, + response: Option<&str>, + messages: Value, + ) -> UsageEventResponse { + claude_request(Some("main"), Some(previous), response, &[], messages) + } + + fn user(text: &str) -> Value { + json!({"id": null, "role": "user", "text": text}) + } + + fn budget() -> Value { + json!({"id": null, "role": "system", "text": "15000000 tokens left"}) + } + + fn tool_result() -> Value { + json!({"id": null, "role": "tool", "text": ""}) + } + + /// The replayed form of a response: thinking is redacted. + fn replayed(text: &str) -> [Value; 3] { + [ + json!({"id": null, "role": "reasoning", "text": ""}), + json!({"id": null, "role": "assistant", "text": text}), + json!({"id": null, "role": "tool", "text": ""}), + ] + } + + /// A response found only by the response id its exchange declares. + fn streamed(id: &str, text: &str) -> UsageEventResponse { + let conversation = json!({ + "messages": [ + {"id": id, "role": "reasoning", "text": ""}, + {"id": id, "role": "assistant", "text": text}, + {"id": null, "role": "tool", "text": ""}, + ], + }); + let mut response = structured_event("response", &conversation.to_string()); + response.stream_id = None; + response.response_id = Some(id.to_owned()); + response + } + + /// A `thread_not_found` error carries no messages and no response id. + fn thread_not_found() -> UsageEventResponse { + let mut response = structured_event( + "response", + r#"{"messages":[],"completeness":"unsupported"}"#, + ); + response.stream_id = None; + response.text = None; + response + } + + fn texts(items: &[TrajectoryItem]) -> Vec<(&str, usize, &str)> { + items + .iter() + .map(|i| (shown(i), i.stream, i.text.as_deref().unwrap_or_default())) + .collect() + } + + const MAIN_CONTEXT: &str = "x-anthropic-billing-header: cc_version=2.1.293.af3; \ + cc_entrypoint=cli; cch=3ceb5;\nYou are Claude Code.\nMain loop \ + instructions."; + + #[test] + fn a_continuation_appends_new_messages_in_the_stream_of_its_response() { + let search = claude_request( + None, + None, + Some("msg_side"), + &[ + "You are Claude Code.", + "You are an assistant for performing a web search tool use", + ], + json!([user("search the web")]), + ); + let events = vec![ + create("msg_1", json!([user("hello"), budget()])), + streamed("msg_1", "looking"), + search, + streamed("msg_side", "results"), + continue_from("msg_1", Some("msg_2"), json!([tool_result(), budget()])), + streamed("msg_2", "found it"), + continue_from( + "msg_2", + Some("msg_3"), + json!([tool_result(), user("thanks"), budget()]), + ), + ]; + let items = build(timeline(events)).items; + + assert_eq!( + texts(&items), + [ + ("system", 0, MAIN_CONTEXT), + ("user", 0, "hello"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "looking"), + ( + "system", + 1, + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ + cch=3ceb5;\nYou are Claude Code.\nYou are an assistant for performing a web \ + search tool use" + ), + ("user", 1, "search the web"), + ("assistant", 1, "results"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "found it"), + ("user", 0, "thanks"), + ( + "system", + 0, + "15000000 tokens left" + ), + ] + ); + } + + #[test] + fn a_replay_after_a_failed_continuation_adds_no_duplicates() { + let first = user("hello"); + let retry = user("try the wire"); + let [looking_thinking, looking, looking_tool] = replayed("looking"); + let [found_thinking, found, found_tool] = replayed("found it"); + let replay = json!([ + first, + budget(), + looking_thinking, + looking, + looking_tool, + tool_result(), + budget(), + found_thinking, + found, + found_tool, + retry, + budget(), + ]); + let events = vec![ + create("msg_1", json!([first, budget()])), + streamed("msg_1", "looking"), + continue_from("msg_1", Some("msg_2"), json!([tool_result(), budget()])), + streamed("msg_2", "found it"), + continue_from("msg_2", None, json!([retry, budget()])), + thread_not_found(), + create("msg_3", replay), + streamed("msg_3", "no wire port"), + continue_from("msg_3", Some("msg_4"), json!([tool_result(), budget()])), + streamed("msg_4", "compressed"), + ]; + let trajectory = build(timeline(events)); + + assert_eq!( + texts(&trajectory.items), + [ + ("system", 0, MAIN_CONTEXT), + ("user", 0, "hello"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "looking"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "found it"), + ("user", 0, "try the wire"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "no wire port"), + ( + "system", + 0, + "15000000 tokens left" + ), + ("assistant", 0, "compressed"), + ] + ); + assert_eq!(trajectory.events.len(), 10); + } + + #[test] + fn a_continuation_of_an_unseen_response_uses_its_stream_id_or_opens_a_stream() { + let title = || { + claude_request( + None, + None, + Some("msg_title"), + &["You are a title generator."], + json!([user("name it")]), + ) + }; + let events = vec![ + create("msg_1", json!([user("hello"), budget()])), + title(), + continue_from("msg_uncaptured", None, json!([user("next"), budget()])), + ]; + let items = build(timeline(events)).items; + assert_eq!( + texts(&items)[items.len() - 2..], + [ + ("user", 0, "next"), + ( + "system", + 0, + "15000000 tokens left" + ), + ] + ); + + // Without a stream id the continuation opens a stream of its own and + // shows only the messages it carries. + let mut orphan = continue_from("msg_uncaptured", None, json!([user("next")])); + orphan.stream_id = None; + let items = build(timeline(vec![title(), orphan])).items; + assert_eq!(texts(&items)[2..], [("user", 1, "next")]); + } + + #[test] + fn declared_stream_ids_keep_each_thread_apart() { + let request = |stream: &str, messages: &str| { + let mut event = structured_event( + "request", + &format!( + r#"{{"messages":[{{"role":"developer","text":"{stream} rules"}},{messages}]}}"# + ), + ); + event.stream_id = Some(stream.to_owned()); + event + }; + let items = build(timeline(vec![ + request("main", r#"{"role":"user","text":"fix it"}"#), + request("guardian", r#"{"role":"user","text":"review the action"}"#), + request( + "main", + r#"{"role":"user","text":"fix it"},{"role":"user","text":"and test"}"#, + ), + request( + "guardian", + r#"{"role":"user","text":"review the action"},{"role":"user","text":"next action"}"#, + ), + ])) + .items; + + assert_eq!( + texts(&items), + [ + ("developer", 0, "main rules"), + ("user", 0, "fix it"), + ("developer", 1, "guardian rules"), + ("user", 1, "review the action"), + ("user", 0, "and test"), + ("user", 1, "next action"), + ] + ); + } + + #[test] + fn requests_without_a_stream_id_each_open_a_stream_and_repeat_their_history() { + let recap = || { + let mut event = structured_event( + "request", + r#"{"messages":[{"role":"system","text":"rules"},{"role":"user","text":"hello"},{"role":"user","text":"recap"}]}"#, + ); + event.stream_id = None; + event + }; + let items = build(timeline(vec![recap(), recap()])).items; + + assert_eq!( + texts(&items), + [ + ("system", 0, "rules"), + ("user", 0, "hello"), + ("user", 0, "recap"), + ("system", 1, "rules"), + ("user", 1, "hello"), + ("user", 1, "recap"), + ] + ); + } + + #[test] + fn a_response_joins_the_stream_of_the_request_with_its_response_id() { + let mut main = create("msg_main", json!([user("hello")])); + main.stream_id = Some("main".to_owned()); + let side = claude_request( + None, + None, + Some("msg_side"), + &["You are a summarizer."], + json!([user("summarize")]), + ); + // The main exchange answers after the side request was observed. + let items = build(timeline(vec![ + main, + side, + streamed("msg_main", "hi"), + streamed("msg_side", "summary"), + streamed("msg_unpaired", "orphan"), + ])) + .items; + + assert_eq!( + texts(&items)[2..], + [ + ( + "system", + 1, + "x-anthropic-billing-header: cc_version=2.1.293.af3; cc_entrypoint=cli; \ + cch=3ceb5;\nYou are a summarizer." + ), + ("user", 1, "summarize"), + ("assistant", 0, "hi"), + ("assistant", 1, "summary"), + ("assistant", 2, "orphan"), + ] + ); + } + + #[test] + fn opening_instructions_show_only_where_they_change() { + let request = |rules: &str, messages: &str| { + structured_event( + "request", + &format!( + r#"{{"messages":[{{"role":"system","text":"base"}},{{"role":"developer","text":"{rules}"}},{messages}],"instructions_present":true}}"# + ), + ) + }; + let items = build(timeline(vec![ + request("rules v1", r#"{"role":"user","text":"hello"}"#), + request( + "rules v1", + r#"{"role":"user","text":"hello"},{"role":"user","text":"again"}"#, + ), + request( + "rules v2", + r#"{"role":"user","text":"hello"},{"role":"user","text":"again"},{"role":"user","text":"more"}"#, + ), + ])) + .items; + + assert_eq!( + texts(&items), + [ + ("system", 0, "base"), + ("developer", 0, "rules v1"), + ("user", 0, "hello"), + ("user", 0, "again"), + ("developer", 0, "rules v2"), + ("user", 0, "more"), + ] + ); + } + + #[test] + fn system_messages_inside_the_conversation_keep_one_stream() { + let first = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"}],"instructions_present":true}"#; + let second = r#"{"messages":[{"role":"system","text":"instructions"},{"id":null,"role":"user","text":"hello"},{"id":null,"role":"system","text":"reminder one"},{"id":null,"role":"assistant","text":"hi"},{"id":null,"role":"system","text":"reminder two"},{"id":null,"role":"user","text":"next"}],"instructions_present":true}"#; + let items = build(timeline(vec![ + structured_event("request", first), + structured_event("request", second), + ])) + .items; + + assert_eq!( + items + .iter() + .map(|i| (shown(i), i.text.as_deref().unwrap_or_default())) + .collect::>(), + [ + ("system", "instructions"), + ("user", "hello"), + ("system", "reminder one"), + ("assistant", "hi"), + ("system", "reminder two"), + ("user", "next"), + ] + ); + assert!(items.iter().all(|i| i.stream == 0)); + } + + const PROMPT: &str = "You are performing a CONTEXT CHECKPOINT COMPACTION. Create a \ + handoff summary for another LLM that will resume the task."; + const SUMMARY: &str = "## Goal\n\nFinish the report."; + + /// One exchange of a call the collector declares as a compaction. + fn compaction(texts: &[&str], response_id: &str) -> [UsageEventResponse; 2] { + let mut request = user_request(texts); + request.metadata["purpose"] = json!(COMPACTION_PURPOSE); + request.response_id = Some(response_id.to_owned()); + let mut response = event("response", SUMMARY, json!({"purpose": COMPACTION_PURPOSE})); + response.response_id = Some(response_id.to_owned()); + [request, response] + } + + #[test] + fn a_compaction_exchange_forms_one_item_and_its_replay_is_ordinary_input() { + let [request, response] = compaction( + &[ + "write the report", + "date", + PROMPT, + ], + "resp_compact", + ); + let replay = format!("Here is the summary of the earlier conversation:\n{SUMMARY}"); + let events = vec![ + user_request(&["write the report"]), + event("response", "working", json!({})), + request, + response, + user_request(&["cwd", &replay]), + event("response", "continuing", json!({})), + ]; + + let items = build(timeline(events)).items; + + assert_eq!( + items.iter().map(shown).collect::>(), + ["user", "text", "compaction", "user", "user", "text",] + ); + // The new input of the compaction request is the item's input. + assert_eq!( + items[2].input.as_deref(), + Some(&*format!( + "date\n\n{PROMPT}" + )) + ); + // The summary comes from the response of the same exchange. + assert_eq!(items[2].output.as_deref(), Some(SUMMARY)); + assert!(items[2].output_event_pk.is_some()); + assert_ne!(items[2].output_event_pk, Some(items[2].event_pk)); + // The replayed summary is shown like any other new input. + assert_eq!(items[4].text.as_deref(), Some(&*replay)); + } + + #[test] + fn compaction_prompt_text_alone_is_ordinary_input() { + let items = build(timeline(vec![ + user_request(&[PROMPT]), + event("response", SUMMARY, json!({})), + ])) + .items; + assert_eq!( + items.iter().map(shown).collect::>(), + ["user", "text"] + ); + } + + fn timeline(events: Vec) -> SessionTimelineResponse { + let now = Utc::now(); + SessionTimelineResponse { + session: SessionInfo { + session_pk: Uuid::nil(), + user_id: Uuid::nil(), + device_context_id: Uuid::nil(), + agent_name: "codex".to_owned(), + agent_version: None, + session_id: "session".to_owned(), + started_at: now, + ended_at: None, + metadata: json!({}), + title: None, + purpose: None, + parent_session_pk: None, + }, + turns: vec![TurnTimeline { + turn_pk: Uuid::nil(), + turn_index: 1, + started_at: now, + ended_at: None, + events, + }], + } + } + + fn event(event_type: &str, text: &str, metadata: Value) -> UsageEventResponse { + UsageEventResponse { + id: Uuid::now_v7(), + event_id: Uuid::now_v7().to_string(), + user_id: Uuid::nil(), + device_context_id: Uuid::nil(), + host_name: None, + platform: None, + session_pk: Uuid::nil(), + turn_pk: Uuid::nil(), + agent_name: "codex".to_owned(), + agent_version: None, + session_id: "session".to_owned(), + turn_index: 1, + llm_provider: "openai".to_owned(), + llm_model: "gpt-5".to_owned(), + event_type: event_type.to_owned(), + text: (!text.is_empty()).then(|| text.to_owned()), + text_sha256: None, + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 0, + observed_at: Utc::now(), + stream_id: Some("main".to_owned()), + response_id: None, + previous_response_id: None, + metadata, + attachments: Vec::new(), + } + } +} diff --git a/crates/abyss-backend/src/usage/trajectory/conversation.rs b/crates/abyss-backend/src/usage/trajectory/conversation.rs new file mode 100644 index 0000000..a97815d --- /dev/null +++ b/crates/abyss-backend/src/usage/trajectory/conversation.rs @@ -0,0 +1,158 @@ +//! Session trajectory projection from collector-produced structured messages. +//! +//! Each exchange is placed in a stream by the identities the collector +//! declares, compared only for equality: a request continuing a known +//! response joins that response's stream, else a request naming a stream +//! joins it, and any other request opens a stream of its own. A response +//! joins the stream of the request that produced it. + +use std::collections::HashMap; + +use super::{TrajectoryItem, TrajectoryItemKind, UsageEventResponse}; +use crate::usage::conversation::{Conversation, Message}; + +impl Message { + fn matches(&self, other: &Self) -> bool { + self.role == other.role + && self.text == other.text + && match (&self.id, &other.id) { + (Some(left), Some(right)) => left == right, + _ => true, + } + } +} + +/// The stream an event belongs to. +#[derive(Clone, Copy)] +pub(super) struct Placement { + pub(super) stream: usize, + /// The request carries only the messages added since the response it + /// continues. + incremental: bool, +} + +#[derive(Default)] +pub(super) struct ConversationProjection { + streams: Vec, + /// Stream of each declared stream id. + keys: HashMap, + /// Stream of each exchange by the response id it declares. + responses: HashMap, +} + +#[derive(Default)] +struct StreamHistory { + /// Instructions the latest full request opened with. + opening: Vec, + /// Messages after the opening, as the stream has seen them so far. + history: Vec, +} + +impl ConversationProjection { + /// Places an event by the identities its exchange declares. + pub(super) fn place(&mut self, event: &UsageEventResponse) -> Placement { + let request = event.event_type == "request"; + let continued = if request { + event + .previous_response_id + .as_ref() + .and_then(|id| self.responses.get(id).copied()) + } else { + // The request of the same exchange registered the response id. + event + .response_id + .as_ref() + .and_then(|id| self.responses.get(id).copied()) + }; + let stream = match (continued, &event.stream_id) { + (Some(stream), _) => stream, + (None, Some(key)) => { + if let Some(&stream) = self.keys.get(key) { + stream + } else { + let stream = self.open(); + self.keys.insert(key.clone(), stream); + stream + } + } + (None, None) => self.open(), + }; + if let Some(id) = &event.response_id { + self.responses.entry(id.clone()).or_insert(stream); + } + Placement { + stream, + incremental: request && event.previous_response_id.is_some(), + } + } + + fn open(&mut self) -> usize { + self.streams.push(StreamHistory::default()); + self.streams.len().saturating_sub(1) + } + + pub(super) fn project( + &mut self, + event: &UsageEventResponse, + placement: Placement, + ) -> Option> { + let conversation = Conversation::read(&event.metadata)?; + let request = event.event_type == "request"; + // A request that declares its own instructions carries them as its + // first message. Messages without text, such as tool items whose + // content the tool fields carry, take no part in the trajectory. + let mut messages = conversation.messages; + let opening_len = usize::from(request && conversation.instructions_present); + let rest = messages + .split_off(opening_len.min(messages.len())) + .into_iter() + .filter(|message| !message.text.is_empty()) + .collect::>(); + let (opening, messages) = (messages, rest); + if opening.is_empty() && messages.is_empty() { + return None; + } + let stream = placement.stream; + let state = self.streams.get_mut(stream)?; + let item = |message: &Message| { + let mut item = TrajectoryItem::new(TrajectoryItemKind::Message, stream, event); + item.role = Some(message.role.clone()); + item.text = Some(message.text.clone()); + item + }; + let mut added = Vec::new(); + // A response and a continuation carry only messages added since the + // response they continue, so each one is new; the history keeps them + // for a later full request that replays them. A continuation does not + // repeat the conversation's instructions. + if !request || placement.incremental { + for message in messages { + added.push(item(&message)); + state.history.push(message); + } + return Some(added); + } + + // A full request shows its instructions only where they changed, and + // of its history only what the stream has not seen in that order. + for message in &opening { + if !message.text.is_empty() && !state.opening.iter().any(|old| old.matches(message)) { + added.push(item(message)); + } + } + state.opening = opening; + + let mut cursor = 0; + for message in &messages { + let found = state.history[cursor..] + .iter() + .position(|old| old.matches(message)); + match found { + Some(offset) => cursor = cursor.saturating_add(offset).saturating_add(1), + None => added.push(item(message)), + } + } + state.history = messages; + Some(added) + } +} diff --git a/scripts/bench/README.md b/scripts/bench/README.md new file mode 100644 index 0000000..92cad64 --- /dev/null +++ b/scripts/bench/README.md @@ -0,0 +1,18 @@ +# Search benchmark + +`search_bench.py` times `GET /v1/agent-usage/search` on a running backend +against a set of queries covering missing, rare, Chinese, common, multi-term, +paged, and filtered searches. It records the cold first run, the median of +three warm runs, the session total, and a hash of the canonical response, so +two builds on the same database can be compared for speed and identical +results: + +```bash +python3 scripts/bench/search_bench.py --base-url http://127.0.0.1:PORT \ + --token-file TOKEN --out before.json +python3 scripts/bench/search_bench.py --compare before.json after.json +python3 scripts/bench/search_bench.py --phases DB.sqlite --term entry +``` + +`--phases` times the original three SQL phases directly against a database +copy. Run benchmarks against a copy, never a live database. diff --git a/scripts/bench/search_bench.py b/scripts/bench/search_bench.py new file mode 100644 index 0000000..b0efc1a --- /dev/null +++ b/scripts/bench/search_bench.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Benchmark GET /v1/agent-usage/search against a running backend. + +The script starts nothing. Point it at a backend with --base-url and +--token-file; it runs every query in the set, records the first (cold) and +median warm latency, the session total, and a hash of the canonical response +body so two runs can be compared for identical results. + + python3 scripts/bench/search_bench.py \ + --base-url http://127.0.0.1:53792 --token-file run/token \ + --out run/baseline.json + python3 scripts/bench/search_bench.py --compare run/baseline.json run/after.json + +--phases DB times the three SQL phases of the original implementation with +Python's sqlite3 against a database copy, to attribute where time goes. +""" + +import argparse +import hashlib +import json +import sqlite3 +import statistics +import sys +import time +import urllib.parse +import urllib.request + +# Terms chosen from real records: hit counts range from none to ~24k events. +QUERIES = [ + {"name": "missing", "q": "qqvvxxnone7"}, + {"name": "rare", "q": "bm25"}, + {"name": "chinese_run", "q": "帮你尝试不同的思路"}, + {"name": "medium_timeline", "q": "timeline"}, + {"name": "medium_useQuery", "q": "useQuery"}, + {"name": "common_entry", "q": "entry"}, + {"name": "common_dashboard", "q": "dashboard"}, + {"name": "very_common_the", "q": "the"}, + {"name": "and_two_terms", "q": "cargo sqlite"}, + {"name": "and_three_terms", "q": "git status worktree"}, + {"name": "entry_page2", "q": "entry", "page": 2}, + {"name": "entry_page5", "q": "entry", "page": 5}, + {"name": "entry_page_size50", "q": "entry", "page_size": 50}, + {"name": "entry_past_end", "q": "entry", "page": 40}, + {"name": "entry_codex", "q": "entry", "agent_name": ["codex"]}, + {"name": "entry_two_agents", "q": "entry", "agent_name": ["codex", "claude-code"]}, + {"name": "entry_model", "q": "entry", "llm_model": ["claude-opus-5-5"]}, + {"name": "entry_requests", "q": "entry", "event_type": "request"}, + {"name": "entry_responses", "q": "entry", "event_type": "response"}, + {"name": "entry_week", "q": "entry", "from": "2026-10-01T07:00:00Z", "to": "2026-10-08T07:00:00Z"}, + {"name": "the_week_codex", "q": "the", "from": "2026-10-01T07:00:00Z", "agent_name": ["codex"]}, +] + + +def request(base_url, token, query): + params = [] + for key, value in query.items(): + if key == "name": + continue + for item in value if isinstance(value, list) else [value]: + params.append((key, str(item))) + url = f"{base_url}/v1/agent-usage/search?{urllib.parse.urlencode(params)}" + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + started = time.perf_counter() + with urllib.request.urlopen(req, timeout=600) as response: + body = response.read() + return time.perf_counter() - started, body + + +def canonical_hash(body): + payload = json.loads(body) + return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def run(args): + token = open(args.token_file, encoding="utf-8").read().strip() + results = [] + for query in QUERIES: + cold, body = request(args.base_url, token, query) + warm = [] + for _ in range(args.repeat): + elapsed, warm_body = request(args.base_url, token, query) + warm.append(elapsed) + if canonical_hash(warm_body) != canonical_hash(body): + print(f"warning: {query['name']} answered differently between runs", file=sys.stderr) + payload = json.loads(body) + result = { + "name": query["name"], + "query": {k: v for k, v in query.items() if k != "name"}, + "cold_s": round(cold, 4), + "median_s": round(statistics.median(warm), 4), + "total_sessions": payload["total_sessions"], + "items": len(payload["items"]), + "hash": canonical_hash(body), + } + results.append(result) + print(f"{result['name']:20} cold {cold:7.3f}s median {result['median_s']:7.3f}s " + f"sessions {result['total_sessions']:5} items {result['items']}") + with open(args.out, "w", encoding="utf-8") as handle: + json.dump(results, handle, ensure_ascii=False, indent=2) + + +def compare(before_path, after_path): + before = {r["name"]: r for r in json.load(open(before_path, encoding="utf-8"))} + after = {r["name"]: r for r in json.load(open(after_path, encoding="utf-8"))} + print(f"{'query':20} {'before':>9} {'after':>9} {'speedup':>8} same") + for name, old in before.items(): + new = after.get(name) + if new is None: + continue + speedup = old["median_s"] / new["median_s"] if new["median_s"] else float("inf") + same = "yes" if old["hash"] == new["hash"] else "NO" + print(f"{name:20} {old['median_s']:8.3f}s {new['median_s']:8.3f}s {speedup:7.1f}x {same}") + + +def phases(db_path, term): + """Times the original three SQL phases for one term.""" + db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + user_id = db.execute("SELECT id FROM app_users LIMIT 1").fetchone()[0] + match = " AND ".join('"' + part.replace('"', '""') + '"' for part in term.split()) + source = ("FROM usage_events_fts INNER JOIN llm_usage_events e ON e.id = usage_events_fts.event_pk " + "WHERE usage_events_fts MATCH ? AND e.user_id = ?") + + def timed(label, sql, params): + started = time.perf_counter() + rows = db.execute(sql, params).fetchall() + print(f"{label:34} {time.perf_counter() - started:7.3f}s rows {len(rows)}") + return rows + + timed("count sessions", f"SELECT COUNT(DISTINCT e.session_pk) {source}", (match, user_id)) + rows = timed("all matches, ordered", f"SELECT e.session_pk {source} ORDER BY bm25(usage_events_fts), " + "e.observed_at DESC, e.session_pk", (match, user_id)) + page = list(dict.fromkeys(row[0] for row in rows))[:20] + marks = ",".join("?" * len(page)) + snippets = ", ".join(f"snippet(usage_events_fts, {column}, '[', ']', '…', 32)" for column in (4, 7, 8, 5, 6)) + timed("page matches with snippets", f"SELECT e.id, {snippets} {source} AND e.session_pk IN ({marks}) " + "ORDER BY bm25(usage_events_fts), e.observed_at DESC, e.id", (match, user_id, *page)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base-url") + parser.add_argument("--token-file") + parser.add_argument("--out") + parser.add_argument("--repeat", type=int, default=3) + parser.add_argument("--compare", nargs=2, metavar=("BEFORE", "AFTER")) + parser.add_argument("--phases", metavar="DB") + parser.add_argument("--term", default="entry") + args = parser.parse_args() + if args.compare: + compare(*args.compare) + elif args.phases: + phases(args.phases, args.term) + elif args.base_url and args.token_file and args.out: + run(args) + else: + parser.error("pass --base-url, --token-file and --out, or --compare, or --phases") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/blackbox_api.py b/scripts/tests/blackbox_api.py index 1ca3350..e2b6c5d 100755 --- a/scripts/tests/blackbox_api.py +++ b/scripts/tests/blackbox_api.py @@ -10,6 +10,7 @@ import urllib.error import urllib.parse import urllib.request +import uuid from dataclasses import dataclass from typing import Any @@ -68,6 +69,24 @@ def require_status(response: HttpResponse, expected: int, context: str) -> None: ) +def token_usage( + input_tokens: int = 0, + output_tokens: int = 0, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, +) -> dict[str, int]: + """AgentEvent token counters; the total is the sum of the disjoint four.""" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + "reasoning_tokens": reasoning_tokens, + "total_tokens": input_tokens + output_tokens + cache_read_tokens + cache_write_tokens, + } + + def event( event_id: str, session_id: str, @@ -84,19 +103,15 @@ def event( "platform": "linux", "os_version": "blackbox", }, - "agent": {"name": "Codex", "version": "blackbox"}, + "agent": {"name": "codex", "version": "blackbox"}, "session_id": session_id, - "turn_index": 1, - "llm": {"provider": "OpenAI", "model": "gpt-5.5"}, + "stream_id": f"stream-{run_id}", + "response_id": f"response-{run_id}", + "llm": {"provider": "openai", "model": "gpt-5.5"}, "event_type": event_type, "text": text, "token_usage": token_usage, - "metadata": { - "test_run": run_id, - "response_id": f"response-{run_id}", - "previous_response_id": None, - "provider_call_index": 1, - }, + "metadata": {"test_run": run_id}, } if event_type == "request": value["attachments"] = [ @@ -135,7 +150,7 @@ def run(base_url: str, token: str, run_id: str) -> None: session_id, "request", "blackbox request", - {"input_tokens": 12, "cache_read_tokens": 3}, + token_usage(input_tokens=12, cache_read_tokens=3), run_id, ), event( @@ -143,7 +158,7 @@ def run(base_url: str, token: str, run_id: str) -> None: session_id, "response", "blackbox response", - {"output_tokens": 21, "reasoning_tokens": 4, "total_tokens": 25}, + token_usage(output_tokens=21, reasoning_tokens=4), run_id, ), ], @@ -214,6 +229,40 @@ def run(base_url: str, token: str, run_id: str) -> None: IMAGE_BASE64.encode() not in raw_response.body, "raw event responses must not expose attachment content", ) + require( + all( + item.get("stream_id") == f"stream-{run_id}" + and item.get("response_id") == f"response-{run_id}" + and item.get("previous_response_id") is None + for item in raw_events + ), + "events must return the exchange identities they were ingested with", + ) + + # The list projection carries what an event list shows and no content. + list_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + {"session_id": session_id, "limit": 10, "fields": "list"} + ) + list_response = request(base_url, "GET", list_path, token=token) + require_status(list_response, 200, "event list query") + listed_events = list_response.json()["events"] + require( + [item["id"] for item in listed_events] == [item["id"] for item in raw_events], + "event list must return the same events in the same order", + ) + content_fields = {"text", "text_sha256", "metadata", "attachments"} + require( + all(content_fields.isdisjoint(item) for item in listed_events), + "event list items must leave out text, metadata, and attachments", + ) + require( + all( + item["total_tokens"] == by_event_id[item["event_id"]]["total_tokens"] + and item["host_name"] == by_event_id[item["event_id"]]["host_name"] + for item in listed_events + ), + "event list items must keep token counts and device labels", + ) attachments = by_event_id[request_event_id]["attachments"] require(len(attachments) == 1, "request event must expose one attachment record") @@ -247,10 +296,88 @@ def run(base_url: str, token: str, run_id: str) -> None: require(len(rows) == 1, "usage summary must contain one aggregate row") summary = rows[0] require(summary["user_id"] == OWNER_ID, "summary must belong to the owner") - require(summary["agent_name"] == "codex", "summary must normalize Agent names") - require(summary["llm_provider"] == "openai", "summary must normalize providers") + require(summary["agent_name"] == "codex", "summary must report the Agent name as received") + require(summary["llm_provider"] == "openai", "summary must report the provider as received") require(summary["requests"] == 1 and summary["responses"] == 1, "summary event counts are incorrect") - require(summary["total_tokens"] == 40, "summary token total is incorrect") + require(summary["total_tokens"] == 36, "summary token total is incorrect") + + # Repeating a filter key matches any of its values; filters still combine + # with each other. + other_session_id = f"{session_id}-absent" + any_of_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [ + ("session_id", other_session_id), + ("session_id", session_id), + ("agent_name", "claude-code"), + ("agent_name", "codex"), + ("llm_provider", "anthropic"), + ("llm_provider", "openai"), + ("llm_model", "gpt-5.5"), + ("llm_model", "absent-model"), + ("limit", 10), + ] + ) + any_of_response = request(base_url, "GET", any_of_path, token=token) + require_status(any_of_response, 200, "raw event query with repeated filters") + require( + len(any_of_response.json()["events"]) == 2, + "repeated filter values must match any of the values", + ) + # Agent names match exactly as reported: no case folding or aliases. + for agent in ["Codex", "claude-desktop"]: + exact_agent_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("agent_name", agent)] + ) + exact_agent_response = request(base_url, "GET", exact_agent_path, token=token) + require_status(exact_agent_response, 200, "raw event query with another Agent") + require( + exact_agent_response.json()["events"] == [], + f"agent_name={agent} must not match codex events", + ) + # Providers also match exactly as reported. + folded_provider_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("llm_provider", "OpenAI")] + ) + folded_provider_response = request(base_url, "GET", folded_provider_path, token=token) + require_status(folded_provider_response, 200, "raw event query with a case-folded provider") + require( + folded_provider_response.json()["events"] == [], + "llm_provider=OpenAI must not match the reported openai", + ) + none_of_path = "/v1/agent-usage/events?" + urllib.parse.urlencode( + [("session_id", session_id), ("llm_model", "absent-model"), ("llm_model", "other-model")] + ) + none_of_response = request(base_url, "GET", none_of_path, token=token) + require_status(none_of_response, 200, "raw event query with unmatched models") + require( + none_of_response.json()["events"] == [], + "filters must still combine across dimensions", + ) + any_of_summary_path = "/v1/agent-usage/summary?" + urllib.parse.urlencode( + [ + ("group_by", "model"), + ("session_id", session_id), + ("session_id", other_session_id), + ("llm_model", "absent-model"), + ("llm_model", "gpt-5.5"), + ] + ) + any_of_summary = request(base_url, "GET", any_of_summary_path, token=token) + require_status(any_of_summary, 200, "usage summary with repeated filters") + any_of_rows = any_of_summary.json()["rows"] + require( + [(row["llm_model"], row["total_tokens"]) for row in any_of_rows] == [("gpt-5.5", 36)], + "summary must aggregate every repeated filter value", + ) + any_of_sessions_path = "/v1/agent-usage/sessions?" + urllib.parse.urlencode( + [("llm_model", "absent-model"), ("llm_model", "gpt-5.5"), ("page_size", 100)] + ) + any_of_sessions = request(base_url, "GET", any_of_sessions_path, token=token) + require_status(any_of_sessions, 200, "session list with repeated filters") + require( + session_id in {item["session_id"] for item in any_of_sessions.json()["items"]}, + "session list must keep sessions that used any selected model", + ) session_pk = raw_events[0]["session_pk"] timeline = request( @@ -265,14 +392,122 @@ def run(base_url: str, token: str, run_id: str) -> None: for turn in timeline.json()["turns"] for item in turn["events"] ] + trajectory = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{session_pk}/trajectory", + token=token, + ) + require_status(trajectory, 200, "session trajectory") + usage_events = trajectory.json()["events"] + require( + [item["event_pk"] for item in usage_events] + == [item["id"] for item in timeline_events], + "trajectory usage must preserve every event in authoritative order", + ) + for usage_event, timeline_event in zip(usage_events, timeline_events): + for field in ( + "turn_index", "event_type", "observed_at", "llm_model", + "input_tokens", "output_tokens", "cache_read_tokens", + "cache_write_tokens", "reasoning_tokens", "total_tokens", + ): + require( + usage_event[field] == timeline_event[field], + f"trajectory usage {field} must match the authoritative event", + ) + require(usage_event["token_usage_available"], "nonzero usage must be available") + require("text" not in usage_event and "metadata" not in usage_event, + "trajectory usage summaries must omit content bodies") require( [item["event_id"] for item in timeline_events] == [request_event_id, response_event_id], "session timeline must preserve request-before-response ordering", ) + outline = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{session_pk}/outline", + token=token, + ) + require_status(outline, 200, "session outline") + outline_body = outline.json() + require( + outline_body["session"] == timeline.json()["session"], + "session outline must return the timeline session information", + ) + outline_events = [ + item for turn in outline_body["turns"] for item in turn["events"] + ] + require( + [item["id"] for item in outline_events] + == [item["id"] for item in timeline_events], + "session outline must preserve the timeline event order", + ) + for outline_event, timeline_event in zip(outline_events, timeline_events): + require( + "text" not in outline_event and "metadata" not in outline_event, + "session outline events must omit content bodies", + ) + for field in ("event_type", "llm_model", "total_tokens", "observed_at"): + require( + outline_event[field] == timeline_event[field], + f"session outline {field} must match the timeline event", + ) + require( + outline_event["text_chars"] == len(timeline_event["text"] or ""), + "session outline text length must count characters", + ) + require( + outline_event["attachment_count"] == len(timeline_event["attachments"]), + "session outline must count event attachments", + ) + missing_outline = request( + base_url, + "GET", + f"/v1/agent-usage/sessions/{uuid.uuid4()}/outline", + token=token, + ) + require_status(missing_outline, 404, "unknown session outline") + + for timeline_event in timeline_events: + single = request( + base_url, + "GET", + f"/v1/agent-usage/events/{timeline_event['id']}", + token=token, + ) + require_status(single, 200, "single event query") + require( + single.json() == timeline_event, + "single event query must match the timeline event exactly", + ) + require( + IMAGE_BASE64.encode() not in single.body, + "single event responses must not expose attachment content", + ) + missing_event = request( + base_url, + "GET", + f"/v1/agent-usage/events/{uuid.uuid4()}", + token=token, + ) + require_status(missing_event, 404, "unknown single event query") + unauthenticated_event = request( + base_url, + "GET", + f"/v1/agent-usage/events/{timeline_events[0]['id']}", + ) + require_status(unauthenticated_event, 401, "unauthenticated single event query") + search_path = "/v1/agent-usage/search?" + urllib.parse.urlencode( - {"q": "blackbox response", "agent_name": "codex"} + [ + ("q", "blackbox response"), + ("agent_name", "claude-code"), + ("agent_name", "codex"), + ("llm_model", "absent-model"), + ("llm_model", "gpt-5.5"), + ] ) search_result: dict[str, Any] = {} for _attempt in range(100): @@ -297,6 +532,13 @@ def run(base_url: str, token: str, run_id: str) -> None: "search must return structured highlights", ) + from session_titles_api import run_title_contract + + run_title_contract(base_url, token, run_id) + + from turns_api import run_turn_contract + + run_turn_contract(base_url, token, run_id) print(f"blackbox API contract passed for {session_id}") diff --git a/scripts/tests/mock_elasticsearch.py b/scripts/tests/mock_elasticsearch.py index a46fad9..1e907a5 100755 --- a/scripts/tests/mock_elasticsearch.py +++ b/scripts/tests/mock_elasticsearch.py @@ -17,8 +17,6 @@ HIGHLIGHT_END = "[[[ABYSS_SEARCH_HIGHLIGHT_END]]]" SEARCHABLE_FIELDS = ( "content", - "commands", - "file_paths", "tool_names", "tool_content", "session_id", @@ -149,6 +147,10 @@ def matches_filters(document: dict[str, Any], filters: list[dict[str, Any]]) -> for field, expected in clause["term"].items(): if document.get(field) != expected: return False + if "terms" in clause: + for field, accepted in clause["terms"].items(): + if document.get(field) not in accepted: + return False if "range" in clause: for field, bounds in clause["range"].items(): value = document.get(field) diff --git a/scripts/tests/session_titles_api.py b/scripts/tests/session_titles_api.py new file mode 100644 index 0000000..0cc9bba --- /dev/null +++ b/scripts/tests/session_titles_api.py @@ -0,0 +1,97 @@ +"""Session title contract shared by both storage black-box suites.""" + +import json +import urllib.parse +import uuid + +from blackbox_api import event, request, require, require_status, token_usage + + +PROMPT = "Draw a pelican riding a bicycle" +CODEX_NAMING = "Generate a concise, single-line task title\nUser prompt:\n" + PROMPT + + +def run_title_contract(base_url: str, token: str, run_id: str) -> None: + def make(session, kind, text, *, naming_thread=False, agent="codex", metadata=None): + value = event(str(uuid.uuid4()), f"{run_id}-{session}", kind, text, token_usage(), run_id) + value.pop("attachments", None) + value["agent"]["name"] = agent + value["device"]["host_name"] = f"title-contract-{run_id}" + value["observed_at"] = "2026-09-30T00:00:01Z" if kind == "request" else "2026-09-30T00:00:02Z" + value["metadata"] = dict(metadata or {}) + if naming_thread: + value["naming_thread"] = True + if kind == "request": + value["metadata"]["conversation"] = { + "messages": [{"role": "user", "text": text}], "completeness": "complete", + } + return value + + def ingest(*events): + result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) + require_status(result, 200, "title fixture ingest") + + def session(name): + query = urllib.parse.urlencode({"session_id": f"{run_id}-{name}", "limit": 1}) + result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) + require_status(result, 200, "title event lookup") + pk = result.json()["events"][0]["session_pk"] + result = request(base_url, "GET", f"/v1/agent-usage/sessions/{pk}/outline", token=token) + require_status(result, 200, "title outline") + return result.json()["session"] + + # Claude Code: the collector reports the naming call and its title, in + # either arrival order relative to the session's first user input. + naming = {"purpose": "session_title"} + for name, title_first in [("claude", False), ("claude-late-input", True)]: + events = [ + make(name, "request", "fix the build", agent="claude-code"), + make(name, "request", "fix the build", agent="claude-code", metadata=naming), + make(name, "response", '{"title":"Build fix"}', agent="claude-code", + metadata={**naming, "session_title": "Build fix"}), + ] + if title_first: + events.reverse() + for value in events: + ingest(value) + value = session(name) + require(value["title"] == "Build fix", f"{name}: the collector title names the session") + require(value["purpose"] is None, f"{name}: a session naming itself is not a title thread") + + # A thread opened only to name another session is marked; the task keeps + # its first user input. + ingest(make("naming", "request", CODEX_NAMING, naming_thread=True), + make("naming", "response", '{"title":"Draw a pelican"}', naming_thread=True)) + value = session("naming") + require(value["purpose"] == "title_generation", "naming threads are marked") + require(value["title"] is None and value["parent_session_pk"] is None, "naming threads are not linked") + ingest(make("task", "request", PROMPT)) + require(session("task")["title"] == PROMPT, "Codex tasks keep their first user input") + + # Instruction text and title-shaped replies never name a session by themselves. + for agent, name in [("claude-code", "quoted-claude"), ("cursor", "generic-codex")]: + ingest(make(name, "request", CODEX_NAMING, agent=agent), + make(name, "response", '{"title":"Build fix"}', agent=agent)) + value = session(name) + require(value["title"] == " ".join(CODEX_NAMING.split())[:79] + "…", f"{agent}/{name} keeps its first input") + require(value["purpose"] is None, f"{agent}/{name} is not a title thread") + + first = make("default", "request", " 修复\n登录问题 " + "好" * 90, agent="cursor") + ingest(first) + expected = "修复 登录问题 " + "好" * 71 + "…" + require(session("default")["title"] == expected, "default title truncates Unicode user input") + later = make("default", "request", "Later request", agent="cursor") + later["observed_at"] = "2026-09-30T00:00:03Z" + ingest(later, make("default", "response", '{"title":"Do not rename"}', agent="cursor")) + require(session("default")["title"] == expected, "later requests and responses preserve the first input") + earlier = make("default", "request", "Earlier request", agent="cursor") + earlier["observed_at"] = "2026-09-30T00:00:00Z" + ingest(earlier) + require(session("default")["title"] == "Earlier request", "late delivery restores the earliest input") + + result = request(base_url, "GET", "/v1/agent-usage/sessions?agent_name=cursor&page_size=100", token=token) + require_status(result, 200, "default title session list") + pk = session("default")["session_pk"] + listed = next(item for item in result.json()["items"] if item["session_pk"] == pk) + require(listed["title"] == "Earlier request", "session list and outline share the default title") + print("session title API contract passed") diff --git a/scripts/tests/turns_api.py b/scripts/tests/turns_api.py new file mode 100644 index 0000000..f43ee26 --- /dev/null +++ b/scripts/tests/turns_api.py @@ -0,0 +1,58 @@ +"""Turn placement contract shared by both storage black-box suites.""" + +import urllib.parse + +from blackbox_api import event, request, require, require_status, token_usage + + +def run_turn_contract(base_url: str, token: str, run_id: str) -> None: + def make(session, name, second, turn_id, *, agent="claude-code"): + value = event(f"{run_id}-{name}", session, "request", name, token_usage(input_tokens=1), run_id) + value.pop("attachments", None) + value["agent"]["name"] = agent + value["llm"] = {"provider": "Anthropic", "model": "claude-x"} + value["observed_at"] = f"2026-10-08T00:00:{second:02}Z" + value["metadata"] = {} + if turn_id is not None: + value["turn_id"] = turn_id + return value + + def ingest(*events): + result = request(base_url, "POST", "/v1/agent-usage/events", token=token, json_body={"events": events}) + require_status(result, 200, "turn fixture ingest") + + def turns(session): + query = urllib.parse.urlencode({"session_id": session, "fields": "list", "limit": 100}) + result = request(base_url, "GET", f"/v1/agent-usage/events?{query}", token=token) + require_status(result, 200, "turn event lookup") + events = result.json()["events"] + return {item["event_id"].removeprefix(f"{run_id}-"): item["turn_index"] for item in events} + + # Events naming a turn share it; events naming none join the turn in + # progress; the first named turn adopts the turn opened without one. + session = f"{run_id}-turns" + ingest( + make(session, "probe", 5, None), + make(session, "main-1", 10, "p1"), + make(session, "subagent-1", 12, "p1"), + make(session, "recap", 20, None), + make(session, "main-2", 30, "p2"), + ) + ingest( + make(session, "late-title", 32, None), + make(session, "late-subagent", 45, "p1"), + ) + turn = turns(session) + for name in ["probe", "main-1", "subagent-1", "recap", "late-subagent"]: + require(turn[name] == 1, f"{name} belongs to the first turn") + for name in ["main-2", "late-title"]: + require(turn[name] == 2, f"{name} belongs to the second turn") + + # Turns are numbered by start time, whatever order they arrive in, and + # without regard to the Agent. + late = f"{run_id}-late-turns" + ingest(make(late, "b", 30, "b", agent="codex"), make(late, "c", 50, "c", agent="codex")) + ingest(make(late, "a", 10, "a", agent="codex")) + turn = turns(late) + require((turn["a"], turn["b"], turn["c"]) == (1, 2, 3), "a late turn takes its place by start time") + print("turn placement API contract passed")