From 4bd90c01f3fcc069996fce190d80538891a4f4a8 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 11:56:47 +0530 Subject: [PATCH 01/10] llm: add typed extensions channel to LlmRequest Provider-specific request options currently have nowhere to live: any new knob forces a typed field on LlmRequest or RequestOptions that every provider then sees. Add Extensions, a TypeId-keyed side channel carried on every request, so codecs can pick up the options meant for them. For now nothing inserts anything; provider_order and session_id keep their typed fields. Wiring happens over the next commits. --- crates/alan/src/main.rs | 1 + crates/llm/src/apis/chat_completions/codec.rs | 1 + crates/llm/src/apis/chat_completions/mod.rs | 1 + crates/llm/src/apis/interactions/codec.rs | 1 + crates/llm/src/apis/interactions/mod.rs | 1 + crates/llm/src/extensions.rs | 75 +++++++++++++++++++ crates/llm/src/lib.rs | 2 + crates/llm/src/request.rs | 4 +- crates/llm/tests/interactions_live.rs | 4 + crates/providers/src/model.rs | 11 ++- 10 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 crates/llm/src/extensions.rs diff --git a/crates/alan/src/main.rs b/crates/alan/src/main.rs index ef44bd3..3a3a945 100644 --- a/crates/alan/src/main.rs +++ b/crates/alan/src/main.rs @@ -198,6 +198,7 @@ fn select_model( server_tools, reasoning_effort: settings.reasoning.unwrap_or_default(), provider_order: settings.provider_order(&model_id), + extensions: llm::Extensions::default(), }, )?) } diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index 2603eaa..ef7c1ac 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -393,6 +393,7 @@ mod tests { credential: None, reasoning_effort: crate::ReasoningEffort::None, provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index f804c14..edfe4d3 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -162,6 +162,7 @@ mod tests { credential: None, reasoning_effort: crate::ReasoningEffort::None, provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/interactions/codec.rs b/crates/llm/src/apis/interactions/codec.rs index a4a8ee0..e583059 100644 --- a/crates/llm/src/apis/interactions/codec.rs +++ b/crates/llm/src/apis/interactions/codec.rs @@ -317,6 +317,7 @@ mod tests { credential: None, reasoning_effort: ReasoningEffort::None, provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/interactions/mod.rs b/crates/llm/src/apis/interactions/mod.rs index e03d9f5..4c9776f 100644 --- a/crates/llm/src/apis/interactions/mod.rs +++ b/crates/llm/src/apis/interactions/mod.rs @@ -390,6 +390,7 @@ mod tests { credential: None, reasoning_effort: crate::ReasoningEffort::None, provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/extensions.rs b/crates/llm/src/extensions.rs new file mode 100644 index 0000000..43db492 --- /dev/null +++ b/crates/llm/src/extensions.rs @@ -0,0 +1,75 @@ +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use std::sync::Arc; +/// Type-erased side-channel carrying provider-specific options on an +/// [`LlmRequest`](crate::LlmRequest). Keys are types, so a value's meaning is +/// defined by whoever defines the key type. +/// +/// Keys live next to their consumers: generic keys (like +/// [`SessionId`]) in `llm`, provider-specific ones (like OpenRouter routing +/// options) in `providers`. The container never inspects what it stores, so +/// the dependency direction `llm <- providers <- agent <- alan` is preserved. +/// +/// Cloning is cheap: values live behind an [`Arc`], so requests can carry +/// their own copy. Insertion is intended before a request is shared. +#[derive(Clone, Default)] +pub struct Extensions { + map: HashMap>, +} + +impl Extensions { + /// Store `value` under its type, replacing any previous value of the same + /// type. + pub fn insert(&mut self, value: T) { + self.map.insert(TypeId::of::(), Arc::new(value)); + } + + /// Borrow the value stored under type `T`, if any. + pub fn get(&self) -> Option<&T> { + self.map + .get(&TypeId::of::()) + .and_then(|value| value.as_ref().downcast_ref::()) + } +} + +/// Canonical session identity, set by the agent for every request. Codecs +/// translate it per provider: OpenRouter sends it as `session_id` and +/// `prompt_cache_key`, vanilla OpenAI-compatible APIs as `prompt_cache_key`. +pub struct SessionId(pub String); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stores_and_returns_typed_values() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("abc".into())); + + assert_eq!(extensions.get::().unwrap().0, "abc"); + assert!(extensions.get::().is_none()); + } + + #[test] + fn replacing_a_key_overwrites_the_value() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("one".into())); + extensions.insert(SessionId("two".into())); + + assert_eq!(extensions.get::().unwrap().0, "two"); + } + + #[test] + fn clones_share_values_without_observing_each_other() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("abc".into())); + let clone = extensions.clone(); + + assert_eq!(clone.get::().unwrap().0, "abc"); + + let mut detached = extensions.clone(); + detached.insert(SessionId("other".into())); + assert_eq!(extensions.get::().unwrap().0, "abc"); + assert_eq!(detached.get::().unwrap().0, "other"); + } +} diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index b75633a..71e1c2e 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -3,6 +3,7 @@ mod api; mod error; mod event; +mod extensions; mod message; mod request; mod response; @@ -15,6 +16,7 @@ pub use api::LlmApi; pub use apis::{ChatCompletionsApi, InteractionsApi}; pub use error::LlmError; pub use event::{LlmEvent, LlmStream}; +pub use extensions::{Extensions, SessionId}; pub use message::{ContentPart, ImageUrl, Message, Role}; pub use request::{ CompletionInput, Credential, LlmRequest, PromptCacheControl, PromptCacheControlType, diff --git a/crates/llm/src/request.rs b/crates/llm/src/request.rs index d2ca4b1..475f59a 100644 --- a/crates/llm/src/request.rs +++ b/crates/llm/src/request.rs @@ -1,4 +1,4 @@ -use crate::{Message, ToolSpec}; +use crate::{Extensions, Message, ToolSpec}; use serde::{Deserialize, Serialize}; use std::fmt; @@ -123,4 +123,6 @@ pub struct LlmRequest<'a> { /// Ordered provider list forwarded to the API. `None` means the request /// carries no `provider` block at all. pub provider_order: Option<&'a [String]>, + /// Provider-specific options, read by codecs when building the wire body. + pub extensions: Extensions, } diff --git a/crates/llm/tests/interactions_live.rs b/crates/llm/tests/interactions_live.rs index 99196fd..e07c67d 100644 --- a/crates/llm/tests/interactions_live.rs +++ b/crates/llm/tests/interactions_live.rs @@ -42,6 +42,7 @@ async fn live_text_completion() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -82,6 +83,7 @@ async fn live_function_call() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -125,6 +127,7 @@ async fn live_function_call_round_trip() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, + extensions: llm::Extensions::default(), }; let first_response = api.complete(first_request).await.unwrap(); let calls: Vec<_> = first_response.tool_calls().cloned().collect(); @@ -164,6 +167,7 @@ async fn live_function_call_round_trip() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, + extensions: llm::Extensions::default(), }; let second_response = api.complete(second_request).await.unwrap(); assert!( diff --git a/crates/providers/src/model.rs b/crates/providers/src/model.rs index f6b8793..5f1c2d6 100644 --- a/crates/providers/src/model.rs +++ b/crates/providers/src/model.rs @@ -1,8 +1,8 @@ use crate::auth::{AuthError, AuthResolver}; use crate::catalog::ModelInfo; use llm::{ - CompletionInput, LlmApi, LlmError, LlmRequest, LlmResponse, LlmStream, ReasoningEffort, - ServerTool, ToolSpec, + CompletionInput, Extensions, LlmApi, LlmError, LlmRequest, LlmResponse, LlmStream, + ReasoningEffort, ServerTool, ToolSpec, }; use std::sync::Arc; use thiserror::Error; @@ -22,6 +22,8 @@ pub struct ModelOptions { /// Ordered provider list forwarded to the API. Empty means the request /// carries no `provider` block at all. pub provider_order: Vec, + /// Provider-specific options forwarded to codecs with every request. + pub extensions: llm::Extensions, } impl From<&Model> for ModelOptions { @@ -30,6 +32,7 @@ impl From<&Model> for ModelOptions { server_tools: value.server_tools.clone(), reasoning_effort: value.reasoning_effort(), provider_order: value.provider_order.clone(), + extensions: value.extensions.clone(), } } } @@ -42,6 +45,7 @@ pub struct Model { server_tools: Vec, reasoning_effort: ReasoningEffort, provider_order: Vec, + extensions: Extensions, } impl Model { @@ -58,6 +62,7 @@ impl Model { server_tools: options.server_tools, reasoning_effort: options.reasoning_effort, provider_order: options.provider_order, + extensions: options.extensions, } } @@ -100,6 +105,7 @@ impl Model { reasoning_effort: self.reasoning_effort, provider_order: (!self.provider_order.is_empty()) .then_some(self.provider_order.as_slice()), + extensions: self.extensions.clone(), }; Ok(self.api.complete(request).await?) } @@ -116,6 +122,7 @@ impl Model { reasoning_effort: self.reasoning_effort, provider_order: (!self.provider_order.is_empty()) .then_some(self.provider_order.as_slice()), + extensions: self.extensions.clone(), }; Ok(self.api.stream(request).await?) } From 84ef9292936ac1d5585b47ebff14190a20f7e417 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 12:10:05 +0530 Subject: [PATCH 02/10] llm: introduce LlmApiCodec and wire ChatCompletionsApi through it The codec functions are hardwired into the API, so a provider cannot change the wire format without forking the whole API struct. Split the two concerns: LlmApiCodec owns request serialization and chunk decoding, ChatCompletionsApi owns transport, SSE framing, and cross-frame state. DefaultChatCompletionsCodec implements the trait over the existing wire code, so the default path is unchanged. Providers attach their own codec via with_codec; wire structs stay private to each codec, the contract is just a body string in and CodecChunks out. --- crates/llm/src/apis/chat_completions/codec.rs | 32 +++++- crates/llm/src/apis/chat_completions/mod.rs | 99 ++++++++++++++++++- crates/llm/src/codec.rs | 30 ++++++ crates/llm/src/lib.rs | 2 + crates/llm/tests/interactions_live.rs | 8 +- 5 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 crates/llm/src/codec.rs diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index ef7c1ac..393eb9a 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -1,6 +1,27 @@ +use crate::codec::{CodecChunk, LlmApiCodec}; use crate::{LlmError, LlmEvent, LlmRequest, Message, Role, StopReason, ToolSpec, Usage}; use serde::{Deserialize, Serialize}; +/// Codec for the canonical OpenAI-compatible `/chat/completions` protocol. +pub struct DefaultChatCompletionsCodec; + +impl LlmApiCodec for DefaultChatCompletionsCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + serialize_request(request) + } + + fn response(&self, data: &str) -> Result { + let chunk = deserialize_stream_chunk(data)?; + let events = stream_events(&chunk); + Ok(CodecChunk { + model: chunk.model, + finish_reason: chunk.finish_reason, + usage: chunk.usage, + events, + }) + } +} + /// Providers to prioritize, in order. Sent only when the caller configures one; /// otherwise the request carries no `provider` block at all. #[derive(Serialize)] @@ -155,7 +176,7 @@ pub(crate) struct StreamChunk { pub(crate) usage: Option, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct StreamToolCall { pub(crate) index: usize, pub(crate) id: Option, @@ -259,10 +280,10 @@ pub(crate) fn deserialize_stream_chunk(body: &str) -> Result Vec { +pub(crate) fn stream_events(chunk: &StreamChunk) -> Vec { let mut events = Vec::new(); - let details = chunk.reasoning_details; - if let Some(reasoning) = chunk.reasoning { + let details = chunk.reasoning_details.clone(); + if let Some(reasoning) = chunk.reasoning.clone() { let details = if details.is_empty() { vec![serde_json::json!({ "type": "reasoning.text", @@ -278,12 +299,13 @@ pub(crate) fn stream_events(chunk: StreamChunk) -> Vec { events.push(LlmEvent::ReasoningDelta { reasoning, details }); } } - if let Some(text) = chunk.text { + if let Some(text) = chunk.text.clone() { events.push(LlmEvent::TextDelta { text }); } events.extend( chunk .tool_calls + .clone() .into_iter() .map(|call| LlmEvent::ToolCallDelta { index: call.index, diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index edfe4d3..e0f8c8c 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -1,6 +1,7 @@ mod codec; use crate::apis::sse::SseDecoder; +use crate::codec::{CodecChunk, LlmApiCodec}; use crate::{HttpClient, LlmApi, LlmError, LlmEvent, LlmRequest, LlmStream, StopReason}; use async_trait::async_trait; use futures_util::{StreamExt, stream}; @@ -12,6 +13,7 @@ const ENDPOINT: &str = "/chat/completions"; pub struct ChatCompletionsApi { base_url: String, http: Arc, + codec: Arc, } impl ChatCompletionsApi { @@ -19,14 +21,22 @@ impl ChatCompletionsApi { Self { base_url: base_url.into(), http, + codec: Arc::new(codec::DefaultChatCompletionsCodec), } } + + /// Attach a provider codec that translates between canonical requests + /// and this provider's wire format. + pub fn with_codec(mut self, codec: Arc) -> Self { + self.codec = codec; + self + } } #[async_trait] impl LlmApi for ChatCompletionsApi { async fn stream(&self, request: LlmRequest<'_>) -> Result { - let body = codec::serialize_request(&request)?; + let body = self.codec.request(&request)?; let url = format!("{}{}", self.base_url.trim_end_matches('/'), ENDPOINT); let response = self.http.post(&url, &body, request.credential).await?; let status = response.status(); @@ -38,7 +48,7 @@ impl LlmApi for ChatCompletionsApi { }); } - let state = StreamState::new(response.bytes_stream()); + let state = StreamState::new(response.bytes_stream(), self.codec.clone()); let output = stream::try_unfold(state, |mut state| async move { let event = state.next_event().await?; Ok::<_, LlmError>(event.map(|event| (event, state))) @@ -51,6 +61,7 @@ impl LlmApi for ChatCompletionsApi { struct StreamState { input: S, decoder: SseDecoder, + codec: Arc, pending: VecDeque, model: Option, stop_reason: Option, @@ -62,10 +73,11 @@ impl StreamState where S: futures_util::Stream> + Unpin, { - fn new(input: S) -> Self { + fn new(input: S, codec: Arc) -> Self { Self { input, decoder: SseDecoder::default(), + codec, pending: VecDeque::new(), model: None, stop_reason: None, @@ -115,7 +127,7 @@ where continue; } - let chunk = codec::deserialize_stream_chunk(&payload)?; + let chunk: CodecChunk = self.codec.response(&payload)?; if let Some(model) = chunk.model.clone() { self.model = Some(model); } @@ -130,7 +142,7 @@ where // Emit usage *after* the chunk's text/reasoning/tool-call deltas // so consumers see content before the summary. let usage_event = chunk.usage.clone().map(|usage| LlmEvent::Usage { usage }); - self.pending.extend(codec::stream_events(chunk)); + self.pending.extend(chunk.events); if let Some(event) = usage_event { self.pending.push_back(event); } @@ -142,6 +154,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::codec::CodecChunk; use crate::{Message, RequestOptions, ToolSpec}; use futures_util::StreamExt; use std::sync::Arc; @@ -213,4 +226,80 @@ mod tests { assert_eq!(response.usage.unwrap().output_tokens, 3); assert!(server.await.unwrap().contains("\"stream\":true")); } + + /// A custom codec replaces both directions: the body it serializes is + /// sent, and the chunk it decodes drives the emitted events. + #[tokio::test] + async fn with_codec_uses_the_custom_codec() { + struct UppercaseCodec; + + impl LlmApiCodec for UppercaseCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + Ok(format!( + r#"{{"model":"{}","upper":true}}"#, + request.model_id + )) + } + + fn response(&self, _data: &str) -> Result { + Ok(CodecChunk { + model: Some("rewired".into()), + finish_reason: Some("stop".into()), + usage: None, + events: vec![LlmEvent::TextDelta { + text: "decoded by custom codec".into(), + }], + }) + } + } + + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 4096]; + loop { + let read = socket.read(&mut buffer).await.unwrap(); + assert!(read > 0); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\ndata: anything\n\ndata: [DONE]\n\n", + ) + .await + .unwrap(); + String::from_utf8(request).unwrap() + }); + + let api = + ChatCompletionsApi::new(format!("http://{address}/v1/"), Arc::new(HttpClient::new())) + .with_codec(Arc::new(UppercaseCodec)); + let messages = [Message::user("ping")]; + let options = RequestOptions::default(); + let mut events = api + .stream(request("model", &messages, &[], &options)) + .await + .unwrap(); + + let mut texts = Vec::new(); + let mut model = None; + let mut done = None; + while let Some(event) = events.next().await { + match event.unwrap() { + LlmEvent::TextDelta { text } => texts.push(text), + LlmEvent::Done { model: served, .. } => model = served, + other => done = Some(format!("{other:?}")), + } + } + + assert_eq!(texts.join(""), "decoded by custom codec"); + assert_eq!(model.as_deref(), Some("rewired")); + assert_eq!(done, None); + assert!(server.await.unwrap().contains(r#""upper":true"#)); + } } diff --git a/crates/llm/src/codec.rs b/crates/llm/src/codec.rs new file mode 100644 index 0000000..e1236b8 --- /dev/null +++ b/crates/llm/src/codec.rs @@ -0,0 +1,30 @@ +use crate::{LlmError, LlmEvent, LlmRequest, Usage}; + +/// One decoded SSE data frame from a provider stream. +#[derive(Debug)] +pub struct CodecChunk { + pub model: Option, + pub finish_reason: Option, + pub usage: Option, + /// Deltas carried by this frame, ready to emit in order. + pub events: Vec, +} + +/// Translates between canonical [`LlmRequest`]s / [`LlmEvent`]s and a +/// provider's wire format. +/// +/// An API's default codec speaks the canonical protocol shape. Providers with +/// extras implement this trait and attach their codec to the API (see +/// [`ChatCompletionsApi::with_codec`](crate::ChatCompletionsApi::with_codec)); +/// provider-specific request options arrive inside +/// [`LlmRequest::extensions`](crate::LlmRequest::extensions). +/// +/// Wire request/response structs stay private to each codec: the API only +/// needs a serialized body and decoded chunks. +pub trait LlmApiCodec: Send + Sync { + /// Serialize the wire body for `request`. + fn request(&self, request: &LlmRequest<'_>) -> Result; + + /// Decode one SSE `data:` frame from the provider stream. + fn response(&self, data: &str) -> Result; +} diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 71e1c2e..6e80962 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -1,6 +1,7 @@ //! Provider-independent LLM protocol types and API implementations. mod api; +mod codec; mod error; mod event; mod extensions; @@ -14,6 +15,7 @@ pub mod apis; pub use api::LlmApi; pub use apis::{ChatCompletionsApi, InteractionsApi}; +pub use codec::{CodecChunk, LlmApiCodec}; pub use error::LlmError; pub use event::{LlmEvent, LlmStream}; pub use extensions::{Extensions, SessionId}; diff --git a/crates/llm/tests/interactions_live.rs b/crates/llm/tests/interactions_live.rs index e07c67d..a658770 100644 --- a/crates/llm/tests/interactions_live.rs +++ b/crates/llm/tests/interactions_live.rs @@ -42,7 +42,7 @@ async fn live_text_completion() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, - extensions: llm::Extensions::default(), + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -83,7 +83,7 @@ async fn live_function_call() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, - extensions: llm::Extensions::default(), + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -127,7 +127,7 @@ async fn live_function_call_round_trip() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, - extensions: llm::Extensions::default(), + extensions: llm::Extensions::default(), }; let first_response = api.complete(first_request).await.unwrap(); let calls: Vec<_> = first_response.tool_calls().cloned().collect(); @@ -167,7 +167,7 @@ async fn live_function_call_round_trip() { credential: Some(&credential), reasoning_effort: ReasoningEffort::None, provider_order: None, - extensions: llm::Extensions::default(), + extensions: llm::Extensions::default(), }; let second_response = api.complete(second_request).await.unwrap(); assert!( From 05734349e83ad30470a24b16fc99431fb0f32c21 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 12:31:19 +0530 Subject: [PATCH 03/10] providers: move openrouter wire extras into OpenRouterCodec OpenRouter's provider routing block, sticky session_id, and usage cost parsing lived inside the generic chat completions codec, so every OpenAI-compatible provider was sent OpenRouter-only fields and cost had nowhere provider-neutral to live. BaseRequest and BaseResponse stay in llm as the canonical wire shapes; OpenRouterCodec flattens them and adds its own fields, then fills Usage cost fields from the flattened usage object. The provider block and session_id now only appear on OpenRouter requests, read from request extensions (openrouter::Options, SessionId) instead of typed fields. zai and local providers use the default codec and their bodies lose the stray provider/session_id keys; OpenRouter's own body is unchanged. --- crates/llm/src/apis/chat_completions/codec.rs | 426 +++++++++--------- crates/llm/src/apis/chat_completions/mod.rs | 7 + crates/providers/src/openrouter.rs | 190 +++++++- 3 files changed, 399 insertions(+), 224 deletions(-) diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index 393eb9a..9eaf5bc 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -7,215 +7,222 @@ pub struct DefaultChatCompletionsCodec; impl LlmApiCodec for DefaultChatCompletionsCodec { fn request(&self, request: &LlmRequest<'_>) -> Result { - serialize_request(request) + serde_json::to_string(&BaseRequest::new(request)).map_err(LlmError::Serialization) } fn response(&self, data: &str) -> Result { - let chunk = deserialize_stream_chunk(data)?; - let events = stream_events(&chunk); - Ok(CodecChunk { - model: chunk.model, - finish_reason: chunk.finish_reason, - usage: chunk.usage, - events, - }) + let response: BaseResponse = serde_json::from_str(data).map_err(LlmError::Serialization)?; + decode_stream_response(&response) } } -/// Providers to prioritize, in order. Sent only when the caller configures one; -/// otherwise the request carries no `provider` block at all. +/// Canonical `/chat/completions` request body. Providers with extras embed it +/// via `#[serde(flatten)]` and add their own fields. #[derive(Serialize)] -struct Request<'a> { - model: &'a str, - stream: bool, - messages: Vec, +pub struct BaseRequest<'a> { + pub model: &'a str, + pub stream: bool, + pub messages: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] - tools: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - temperature: Option, + pub tools: Vec, #[serde(skip_serializing_if = "Option::is_none")] - max_tokens: Option, - reasoning: WireReasoning, + pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] - session_id: Option<&'a str>, + pub max_tokens: Option, + pub reasoning: WireReasoning, #[serde(skip_serializing_if = "Option::is_none")] - prompt_cache_key: Option<&'a str>, + pub prompt_cache_key: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] - cache_control: Option<&'a crate::PromptCacheControl>, - #[serde(skip_serializing_if = "Option::is_none")] - provider: Option, + pub cache_control: Option<&'a crate::PromptCacheControl>, +} + +impl<'a> BaseRequest<'a> { + pub fn new(request: &'a LlmRequest<'a>) -> Self { + Self { + model: request.model_id, + stream: true, + messages: request.messages.iter().map(wire_message).collect(), + tools: request.tools.iter().map(wire_tool).collect(), + temperature: request.options.temperature, + max_tokens: request.options.max_tokens, + reasoning: WireReasoning { + effort: request.reasoning_effort.to_string(), + }, + prompt_cache_key: request.options.prompt_cache_key.as_deref(), + cache_control: request.options.cache_control.as_ref(), + } + } +} + +/// Canonical `/chat/completions` stream chunk. Providers with extra fields +/// embed it via `#[serde(flatten)]` and patch the decoded result. +#[derive(Deserialize)] +pub struct BaseResponse { + pub model: Option, + pub choices: Vec, + pub usage: Option, +} + +/// Decode a canonical [`BaseResponse`] into the API-facing [`CodecChunk`]. +/// Exposed so provider codecs can reuse the canonical mapping after +/// flattening [`BaseResponse`] into their own wire chunk. +pub fn decode_stream_response(response: &BaseResponse) -> Result { + let chunk = stream_chunk_from_response(response)?; + let events = stream_events(&chunk); + Ok(CodecChunk { + model: chunk.model, + finish_reason: chunk.finish_reason, + usage: chunk.usage, + events, + }) } +/// OpenRouter-style routing block: which upstream providers to prefer. #[derive(Serialize)] -struct WireProvider { - only: Vec, - allow_fallbacks: bool, +pub struct WireProvider { + pub only: Vec, + pub allow_fallbacks: bool, } #[derive(Serialize)] -struct WireMessage { - role: &'static str, - content: Option, +pub struct WireMessage { + pub role: &'static str, + pub content: Option, #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, + pub tool_calls: Option>, #[serde(skip_serializing_if = "Option::is_none")] - tool_call_id: Option, + pub tool_call_id: Option, #[serde(skip_serializing_if = "Option::is_none")] - reasoning_details: Option>, + pub reasoning_details: Option>, } #[derive(Serialize)] -struct WireToolCall { - id: String, +pub struct WireToolCall { + pub id: String, #[serde(rename = "type")] - kind: &'static str, - function: WireFunction, + pub kind: &'static str, + pub function: WireFunction, } #[derive(Serialize)] -struct WireFunction { - name: String, - arguments: String, +pub struct WireFunction { + pub name: String, + pub arguments: String, } #[derive(Serialize)] -struct WireTool { +pub struct WireTool { #[serde(rename = "type")] - kind: String, + pub kind: String, #[serde(skip_serializing_if = "Option::is_none")] - function: Option, + pub function: Option, } #[derive(Serialize)] -struct WireDefinition { - name: String, - description: String, - parameters: serde_json::Value, +pub struct WireDefinition { + pub name: String, + pub description: String, + pub parameters: serde_json::Value, } #[derive(Serialize)] -struct WireReasoning { - effort: String, +pub struct WireReasoning { + pub effort: String, } #[derive(Deserialize)] -struct StreamResponse { - model: Option, - choices: Vec, - usage: Option, +pub struct StreamChoice { + pub delta: Option, + pub finish_reason: Option, } #[derive(Deserialize)] -struct StreamChoice { - delta: Option, - finish_reason: Option, +pub struct StreamDelta { + pub content: Option, + pub tool_calls: Option>, + pub reasoning: Option, + pub reasoning_content: Option, + pub reasoning_details: Option>, } #[derive(Deserialize)] -struct StreamDelta { - content: Option, - tool_calls: Option>, - reasoning: Option, - reasoning_content: Option, - reasoning_details: Option>, +pub struct StreamToolCallWire { + pub index: Option, + pub id: Option, + pub function: Option, } #[derive(Deserialize)] -struct StreamToolCallWire { - index: Option, - id: Option, - function: Option, -} - -#[derive(Deserialize)] -struct StreamFunctionWire { - name: Option, - arguments: Option, +pub struct StreamFunctionWire { + pub name: Option, + pub arguments: Option, } +/// Canonical usage fields; the OpenAI-standard token accounting. #[derive(Deserialize, Default)] #[serde(default)] -struct WireUsage { - prompt_tokens: u64, - completion_tokens: u64, - total_tokens: Option, - cost: Option, - cost_details: Option, - prompt_tokens_details: Option, - completion_tokens_details: Option, -} - -#[derive(Deserialize, Clone, Default)] -#[serde(default)] -struct WireCostDetails { - upstream_inference_cost: Option, +pub struct WireUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: Option, + pub prompt_tokens_details: Option, + pub completion_tokens_details: Option, } #[derive(Deserialize, Clone, Default)] #[serde(default)] -struct WirePromptTokensDetails { - cached_tokens: Option, - cache_write_tokens: Option, - audio_tokens: Option, +pub struct WirePromptTokensDetails { + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub audio_tokens: Option, } #[derive(Deserialize, Clone, Default)] #[serde(default)] -struct WireCompletionTokensDetails { - reasoning_tokens: Option, +pub struct WireCompletionTokensDetails { + pub reasoning_tokens: Option, } -#[derive(Debug)] -pub(crate) struct StreamChunk { - pub(crate) model: Option, - pub(crate) text: Option, - pub(crate) reasoning: Option, - pub(crate) reasoning_details: Vec, - pub(crate) tool_calls: Vec, - pub(crate) finish_reason: Option, - pub(crate) usage: Option, -} - -#[derive(Debug, Clone)] -pub(crate) struct StreamToolCall { - pub(crate) index: usize, - pub(crate) id: Option, - pub(crate) name: Option, - pub(crate) arguments: String, +/// Map canonical [`WireUsage`] onto [`Usage`]. +pub fn usage_from_wire(wire: &WireUsage) -> Usage { + Usage { + input_tokens: wire.prompt_tokens, + output_tokens: wire.completion_tokens, + total_tokens: wire.total_tokens, + cost: None, + upstream_inference_cost: None, + cached_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.cached_tokens), + cache_write_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.cache_write_tokens), + reasoning_tokens: wire + .completion_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens), + audio_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.audio_tokens), + } } -pub(crate) fn serialize_request(request: &LlmRequest<'_>) -> Result { - let messages = request.messages.iter().map(wire_message).collect(); - let tools = request.tools.iter().map(wire_tool).collect(); - let reasoning = WireReasoning { - effort: request.reasoning_effort.to_string(), - }; - let provider = request - .provider_order - .filter(|order| !order.is_empty()) - .map(|order| WireProvider { - only: order.to_vec(), - allow_fallbacks: order.is_empty(), - }); - - serde_json::to_string(&Request { - model: request.model_id, - stream: true, - messages, - tools, - temperature: request.options.temperature, - max_tokens: request.options.max_tokens, - reasoning, - session_id: request.options.session_id.as_deref(), - prompt_cache_key: request.options.prompt_cache_key.as_deref(), - cache_control: request.options.cache_control.as_ref(), - provider, +/// Map a canonical finish reason onto [`StopReason`]. +pub fn stop_reason_for_finish_reason(reason: Option<&str>) -> Option { + reason.map(|reason| match reason { + "tool_calls" | "function_call" => StopReason::ToolUse, + "length" => StopReason::Length, + "content_filter" => StopReason::ContentFilter, + "stop" => StopReason::Stop, + _ => StopReason::Error, }) - .map_err(LlmError::Serialization) } -pub(crate) fn deserialize_stream_chunk(body: &str) -> Result { - let response: StreamResponse = serde_json::from_str(body).map_err(LlmError::Serialization)?; +/// Decode a canonical [`BaseResponse`] into the internal stream chunk. +pub(crate) fn stream_chunk_from_response(response: &BaseResponse) -> Result { let choice = response.choices.first(); let delta = choice.and_then(|choice| choice.delta.as_ref()); let text = delta @@ -256,30 +263,35 @@ pub(crate) fn deserialize_stream_chunk(body: &str) -> Result, + pub(crate) text: Option, + pub(crate) reasoning: Option, + pub(crate) reasoning_details: Vec, + pub(crate) tool_calls: Vec, + pub(crate) finish_reason: Option, + pub(crate) usage: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct StreamToolCall { + pub(crate) index: usize, + pub(crate) id: Option, + pub(crate) name: Option, + pub(crate) arguments: String, +} + pub(crate) fn stream_events(chunk: &StreamChunk) -> Vec { let mut events = Vec::new(); let details = chunk.reasoning_details.clone(); @@ -330,16 +342,6 @@ fn reasoning_text(details: &[serde_json::Value]) -> String { .collect() } -pub(crate) fn stop_reason_for_finish_reason(reason: Option<&str>) -> Option { - reason.map(|reason| match reason { - "tool_calls" | "function_call" => StopReason::ToolUse, - "length" => StopReason::Length, - "content_filter" => StopReason::ContentFilter, - "stop" => StopReason::Stop, - _ => StopReason::Error, - }) -} - fn wire_message(message: &Message) -> WireMessage { let content = if let Some(parts) = &message.content_parts { Some(serde_json::to_value(parts).expect("content parts must serialize")) @@ -399,6 +401,10 @@ fn role(role: Role) -> &'static str { #[cfg(test)] mod tests { use super::*; + + fn codec() -> super::DefaultChatCompletionsCodec { + super::DefaultChatCompletionsCodec + } use crate::{RequestOptions, ToolDefinition, ToolSpec}; fn request<'a>( @@ -432,7 +438,9 @@ mod tests { max_tokens: Some(128), ..RequestOptions::default() }; - let body = serialize_request(&request("model-a", &messages, &tools, &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &tools, &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(json["model"], "model-a"); @@ -460,7 +468,7 @@ mod tests { let mut request = request("model-a", &messages, &[], &options); request.reasoning_effort = crate::ReasoningEffort::High; let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); + serde_json::from_str(&codec().request(&request).unwrap()).unwrap(); assert_eq!(json["reasoning"]["effort"], "high"); assert!( json["messages"][0].get("reasoning_details").is_none(), @@ -476,7 +484,9 @@ mod tests { max_tokens: None, ..RequestOptions::default() }; - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); assert!(json.get("tools").is_none()); @@ -484,43 +494,6 @@ mod tests { assert!(json.get("max_tokens").is_none()); } - #[test] - fn omits_provider_block_when_unconfigured() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = None; - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert!(json.get("provider").is_none()); - } - - #[test] - fn omits_provider_block_when_order_is_empty() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = Some(&[]); - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert!(json.get("provider").is_none()); - } - - #[test] - fn serializes_provider_order_when_configured() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let order = vec!["deepseek".to_string(), "fireworks".to_string()]; - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = Some(&order); - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert_eq!( - json["provider"]["only"], - serde_json::json!(["deepseek", "fireworks"]) - ); - } - #[test] fn decodes_text_tool_calls_finish_reason_and_usage() { let body = r#"{ @@ -539,18 +512,30 @@ mod tests { "usage": {"prompt_tokens": 11, "completion_tokens": 7} }"#; - let chunk = deserialize_stream_chunk(body).unwrap(); + let chunk = codec().response(body).unwrap(); assert_eq!(chunk.model.as_deref(), Some("served-model")); - assert_eq!(chunk.text.as_deref(), Some("Checking")); - assert_eq!(chunk.tool_calls[0].name.as_deref(), Some("weather")); - assert_eq!(chunk.tool_calls[0].arguments, "{\"city\":"); + assert_eq!( + chunk.events, + vec![ + LlmEvent::TextDelta { + text: "Checking".into() + }, + LlmEvent::ToolCallDelta { + index: 0, + id: Some("call-1".into()), + name: Some("weather".into()), + arguments: "{\"city\":".into(), + signature: None, + }, + ] + ); assert_eq!(chunk.finish_reason.as_deref(), Some("tool_calls")); assert_eq!(chunk.usage.unwrap().output_tokens, 7); } #[test] fn rejects_tool_call_without_index() { - let error = deserialize_stream_chunk( + let error = codec().response( r#"{"choices":[{"delta":{"tool_calls":[{"id":"call-1","function":{"name":"bash"}}]}}]}"#, ) .unwrap_err(); @@ -559,11 +544,10 @@ mod tests { #[test] fn accepts_usage_only_chunk() { - let chunk = deserialize_stream_chunk( - r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#, - ) - .unwrap(); - assert!(chunk.text.is_none()); + let chunk = codec() + .response(r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#) + .unwrap(); + assert!(chunk.events.is_empty()); assert_eq!(chunk.usage.unwrap().input_tokens, 2); } @@ -594,7 +578,9 @@ mod tests { }, ])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; @@ -626,7 +612,9 @@ mod tests { }, ])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; @@ -642,7 +630,9 @@ mod tests { text: "hello".into(), }])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index e0f8c8c..d5f6996 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -1,5 +1,12 @@ mod codec; +pub use codec::{ + BaseRequest, BaseResponse, DefaultChatCompletionsCodec, StreamChoice, StreamDelta, + StreamFunctionWire, StreamToolCallWire, WireDefinition, WireFunction, WireMessage, + WireProvider, WireReasoning, WireTool, WireToolCall, WireUsage, decode_stream_response, + stop_reason_for_finish_reason, usage_from_wire, +}; + use crate::apis::sse::SseDecoder; use crate::codec::{CodecChunk, LlmApiCodec}; use crate::{HttpClient, LlmApi, LlmError, LlmEvent, LlmRequest, LlmStream, StopReason}; diff --git a/crates/providers/src/openrouter.rs b/crates/providers/src/openrouter.rs index 47cb2b1..e6964fa 100644 --- a/crates/providers/src/openrouter.rs +++ b/crates/providers/src/openrouter.rs @@ -3,13 +3,191 @@ use crate::{ ProviderError, ProviderId, ServerToolInfo, }; use async_trait::async_trait; -use llm::{ChatCompletionsApi, HttpClient, LlmApi}; +use llm::apis::chat_completions::{ + BaseRequest, BaseResponse, WireProvider, WireUsage, decode_stream_response, +}; +use llm::{ChatCompletionsApi, CodecChunk, HttpClient, LlmApi, LlmApiCodec, LlmError, LlmRequest}; use reqwest::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; const BASE_URL: &str = "https://openrouter.ai/api/v1"; +/// OpenRouter's `/chat/completions` wire format: the canonical body plus +/// provider routing and a sticky-session key. +struct OpenRouterCodec; + +/// OpenRouter routing options, attached to a request's extensions by the +/// application. +pub struct Options { + /// Providers to prioritize, in order. + pub provider_order: Vec, +} + +#[derive(Serialize)] +struct OpenRouterRequest<'a> { + #[serde(flatten)] + base: BaseRequest<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + session_id: Option<&'a str>, +} + +#[derive(Deserialize)] +struct OpenRouterStreamChunk { + model: Option, + choices: Vec, + usage: Option, +} + +/// OpenRouter extends the standard usage object with cost fields. +#[derive(Deserialize)] +struct OpenRouterUsage { + #[serde(flatten)] + base: WireUsage, + cost: Option, + cost_details: Option, +} + +#[derive(Deserialize)] +struct CostDetails { + upstream_inference_cost: Option, +} + +impl LlmApiCodec for OpenRouterCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + let provider = request + .extensions + .get::() + .map(|options| options.provider_order.as_slice()) + .filter(|order| !order.is_empty()) + .map(|order| WireProvider { + only: order.to_vec(), + allow_fallbacks: order.is_empty(), + }); + let session_id = request.extensions.get::(); + let wire = OpenRouterRequest { + base: BaseRequest::new(request), + provider, + session_id: session_id.map(|session| session.0.as_str()), + }; + serde_json::to_string(&wire).map_err(LlmError::Serialization) + } + + fn response(&self, data: &str) -> Result { + let chunk: OpenRouterStreamChunk = + serde_json::from_str(data).map_err(LlmError::Serialization)?; + let OpenRouterUsage { + base, + cost, + cost_details, + } = match chunk.usage { + Some(usage) => usage, + None => { + return decode_stream_response(&BaseResponse { + model: chunk.model, + choices: chunk.choices, + usage: None, + }); + } + }; + let mut decoded = decode_stream_response(&BaseResponse { + model: chunk.model, + choices: chunk.choices, + usage: Some(base), + })?; + if let Some(usage) = &mut decoded.usage { + usage.cost = cost; + usage.upstream_inference_cost = cost_details + .as_ref() + .and_then(|details| details.upstream_inference_cost); + } + Ok(decoded) + } +} + +#[cfg(test)] +mod codec_tests { + use super::*; + use llm::{Extensions, Message, RequestOptions}; + + fn wire_request(order: Option>, session: Option<&str>) -> (String, Vec) { + let mut extensions = Extensions::default(); + if let Some(order) = order { + extensions.insert(Options { + provider_order: order, + }); + } + if let Some(session) = session { + extensions.insert(llm::SessionId(session.to_string())); + } + let messages = vec![Message::user("hello")]; + let options = RequestOptions::default(); + let request = LlmRequest { + model_id: "model-a", + messages: &messages, + tools: &[], + options: &options, + credential: None, + reasoning_effort: llm::ReasoningEffort::None, + provider_order: None, + extensions, + }; + let body = OpenRouterCodec.request(&request).unwrap(); + (body, messages) + } + + #[test] + fn serializes_provider_order_and_session_id() { + let (body, _keep) = wire_request( + Some(vec!["deepseek".into(), "fireworks".into()]), + Some("s-1"), + ); + let json: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + json["provider"]["only"], + serde_json::json!(["deepseek", "fireworks"]) + ); + assert_eq!(json["session_id"], "s-1"); + assert_eq!(json["model"], "model-a"); + } + + #[test] + fn omits_provider_and_session_when_unconfigured() { + let (body, _keep) = wire_request(None, None); + let json: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(json.get("provider").is_none()); + assert!(json.get("session_id").is_none()); + } + + #[test] + fn fills_cost_from_response() { + let chunk = OpenRouterCodec + .response( + r#"{"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":7,"cost":0.0021,"cost_details":{"upstream_inference_cost":0.0019}}}"#, + ) + .unwrap(); + let usage = chunk.usage.unwrap(); + assert_eq!(usage.input_tokens, 11); + assert_eq!(usage.output_tokens, 7); + assert_eq!(usage.cost, Some(0.0021)); + assert_eq!(usage.upstream_inference_cost, Some(0.0019)); + } + + #[test] + fn leaves_cost_unset_when_response_has_none() { + let chunk = OpenRouterCodec + .response(r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#) + .unwrap(); + let usage = chunk.usage.unwrap(); + assert_eq!(usage.cost, None); + assert_eq!(usage.upstream_inference_cost, None); + } +} + pub struct OpenRouterProvider { id: ProviderId, models: std::sync::RwLock>, @@ -162,10 +340,10 @@ impl OpenRouterBuilder { pub fn build(self) -> Result { let api = self.api.unwrap_or_else(|| { - Arc::new(ChatCompletionsApi::new( - BASE_URL, - Arc::new(HttpClient::new()), - )) + Arc::new( + ChatCompletionsApi::new(BASE_URL, Arc::new(HttpClient::new())) + .with_codec(Arc::new(OpenRouterCodec)), + ) }); let auth = self .auth From 08da41d98b4f634bc669559bd642ea002907b54c Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 12:41:20 +0530 Subject: [PATCH 04/10] agent: carry provider options in request extensions provider_order and session/prompt-cache keys were typed fields on the canonical request types, so every provider saw OpenRouter concepts. The canonical types now describe only what every chat API shares; anything provider-specific rides in Extensions, keyed by types defined next to their consumers: - llm keeps SessionId, the one cross-provider concept (agent sets it, codecs translate: OpenRouter to session_id + prompt_cache_key, default codec to prompt_cache_key only) - providers keeps OpenRouterOptions (routing order), built by alan from settings and passed at bind time - Model merges model-level and per-request extensions, so both the bound options and the agent's session id reach the codec - Agent::set_provider_order becomes set_extensions, agnostic of provider ModelOptions and CompletionInput carry extensions instead of provider_order; wire behavior for OpenRouter is unchanged and zai keeps sending only prompt_cache_key. --- crates/agent/src/agent/mod.rs | 6 ++--- crates/agent/src/agent/prompt.rs | 7 +++-- crates/alan/src/core/settings.rs | 15 +++++++++++ crates/alan/src/main.rs | 3 +-- .../src/views/components/chat_view/models.rs | 12 ++++++--- .../views/components/chat_view/profiles.rs | 2 +- crates/llm/src/apis/chat_completions/codec.rs | 6 +++-- crates/llm/src/apis/chat_completions/mod.rs | 1 - crates/llm/src/apis/interactions/codec.rs | 1 - crates/llm/src/apis/interactions/mod.rs | 1 - crates/llm/src/extensions.rs | 26 +++++++++++++++++++ crates/llm/src/request.rs | 9 ++----- crates/llm/tests/interactions_live.rs | 4 --- crates/providers/src/lib.rs | 2 +- crates/providers/src/model.rs | 25 +++++++----------- crates/providers/src/openrouter.rs | 1 - 16 files changed, 76 insertions(+), 45 deletions(-) diff --git a/crates/agent/src/agent/mod.rs b/crates/agent/src/agent/mod.rs index 48a2505..ba34321 100644 --- a/crates/agent/src/agent/mod.rs +++ b/crates/agent/src/agent/mod.rs @@ -370,13 +370,13 @@ impl Agent { Ok(()) } - pub async fn set_provider_order(&self, provider_order: Vec) -> Result<(), AgentError> { + pub async fn set_extensions(&self, extensions: llm::Extensions) -> Result<(), AgentError> { let mut model = self.model.try_lock().map_err(|_| { AgentError::Model(providers::ModelError::Llm(llm::LlmError::Configuration( - "agent is busy: cannot change provider order mid-run".into(), + "agent is busy: cannot change request extensions mid-run".into(), ))) })?; - model.set_provider_order(provider_order); + model.set_extensions(extensions); Ok(()) } } diff --git a/crates/agent/src/agent/prompt.rs b/crates/agent/src/agent/prompt.rs index 7751917..e30a573 100644 --- a/crates/agent/src/agent/prompt.rs +++ b/crates/agent/src/agent/prompt.rs @@ -184,11 +184,13 @@ pub(super) async fn summarize( messages.push(Message::user(instruction)); let options = RequestOptions::default(); + let extensions = llm::Extensions::default(); let mut stream_resp = model .stream(CompletionInput { messages: &messages, tools: &[], options: &options, + extensions: &extensions, }) .await?; @@ -217,9 +219,9 @@ pub(super) async fn stream_round( .collect(); let messages = build_messages(context); + let mut extensions = llm::Extensions::default(); + extensions.insert(llm::SessionId(session_id)); let options = RequestOptions { - prompt_cache_key: Some(session_id.clone()), - session_id: Some(session_id), cache_control: Some(PromptCacheControl::one_hour()), ..RequestOptions::default() }; @@ -229,6 +231,7 @@ pub(super) async fn stream_round( messages: &messages, tools: &tools, options: &options, + extensions: &extensions, }) .await?; diff --git a/crates/alan/src/core/settings.rs b/crates/alan/src/core/settings.rs index 04685d8..0cc4c8f 100644 --- a/crates/alan/src/core/settings.rs +++ b/crates/alan/src/core/settings.rs @@ -22,6 +22,7 @@ use crate::core::store::JsonStore; const SETTINGS_KEY: &str = "settings"; use anyhow::Context as _; use llm::ReasoningEffort; +use providers::OpenRouterOptions; use std::collections::BTreeMap; use std::path::PathBuf; @@ -146,6 +147,20 @@ impl Settings { pub fn provider_order(&self, model: &str) -> Vec { self.provider_orders.get(model).cloned().unwrap_or_default() } + + /// Provider-specific request options for `model`, keyed by provider. + /// Only OpenRouter exposes settings today; the order is routed to its + /// codec via request extensions. + pub fn extensions(&self, model: &str) -> llm::Extensions { + let mut extensions = llm::Extensions::default(); + let order = self.provider_order(model); + if !order.is_empty() { + extensions.insert(OpenRouterOptions { + provider_order: order, + }); + } + extensions + } } /// Apply a PatchSettings to mutate only specified fields. diff --git a/crates/alan/src/main.rs b/crates/alan/src/main.rs index 3a3a945..83a6ba4 100644 --- a/crates/alan/src/main.rs +++ b/crates/alan/src/main.rs @@ -197,8 +197,7 @@ fn select_model( ModelOptions { server_tools, reasoning_effort: settings.reasoning.unwrap_or_default(), - provider_order: settings.provider_order(&model_id), - extensions: llm::Extensions::default(), + extensions: settings.extensions(&model_id), }, )?) } diff --git a/crates/alan/src/views/components/chat_view/models.rs b/crates/alan/src/views/components/chat_view/models.rs index fdba1a9..f049b0a 100644 --- a/crates/alan/src/views/components/chat_view/models.rs +++ b/crates/alan/src/views/components/chat_view/models.rs @@ -8,8 +8,8 @@ use crate::root::AlanAction; use alan_tui::TaskError; use alan_tui::context::Context; use providers::{ - Model, ModelInfo, ModelOptions, Provider, ProviderId, ProviderRegistry, bind_local_model, - bind_model, + Model, ModelInfo, ModelOptions, OpenRouterOptions, Provider, ProviderId, ProviderRegistry, + bind_local_model, bind_model, }; use std::sync::Arc; @@ -63,7 +63,7 @@ pub(crate) fn open_models_picker(view: &mut ChatView, cx: &mut Context<'_, ChatV let settings = settings::get_settings() .await .map_err(|e| TaskError(e.into()))?; - options.provider_order = settings.provider_order(&model_info.id); + options.extensions = settings.extensions(&model_info.id); let model = bind_model_info(&providers, &model_info, options)?; let name = model_info.name.clone(); @@ -124,8 +124,12 @@ pub(crate) fn apply_model_provider( cx.spawn( async move { let model_id = agent.info().await.id; + let mut extensions = agent.model_options().await.extensions; + extensions.insert(OpenRouterOptions { + provider_order: provider_order.clone(), + }); agent - .set_provider_order(provider_order.clone()) + .set_extensions(extensions) .await .map_err(|error| TaskError(Box::new(error)))?; diff --git a/crates/alan/src/views/components/chat_view/profiles.rs b/crates/alan/src/views/components/chat_view/profiles.rs index 4a9731e..e166d42 100644 --- a/crates/alan/src/views/components/chat_view/profiles.rs +++ b/crates/alan/src/views/components/chat_view/profiles.rs @@ -187,7 +187,7 @@ fn apply_profile( async move { let mut options = agent.model_options().await; let settings = settings::get_settings().await.map_err(super::to_task)?; - options.provider_order = settings.provider_order(&profile.model); + options.extensions = settings.extensions(&profile.model); options.reasoning_effort = profile.reasoning; options.server_tools = server_tools_for_provider( &providers, diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index 9eaf5bc..37c27f5 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -48,7 +48,10 @@ impl<'a> BaseRequest<'a> { reasoning: WireReasoning { effort: request.reasoning_effort.to_string(), }, - prompt_cache_key: request.options.prompt_cache_key.as_deref(), + prompt_cache_key: request + .extensions + .get::() + .map(|session| session.0.as_str()), cache_control: request.options.cache_control.as_ref(), } } @@ -420,7 +423,6 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index d5f6996..a939beb 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -181,7 +181,6 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/interactions/codec.rs b/crates/llm/src/apis/interactions/codec.rs index e583059..20723b2 100644 --- a/crates/llm/src/apis/interactions/codec.rs +++ b/crates/llm/src/apis/interactions/codec.rs @@ -316,7 +316,6 @@ mod tests { options, credential: None, reasoning_effort: ReasoningEffort::None, - provider_order: None, extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/interactions/mod.rs b/crates/llm/src/apis/interactions/mod.rs index 4c9776f..93533f4 100644 --- a/crates/llm/src/apis/interactions/mod.rs +++ b/crates/llm/src/apis/interactions/mod.rs @@ -389,7 +389,6 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/extensions.rs b/crates/llm/src/extensions.rs index 43db492..bf20396 100644 --- a/crates/llm/src/extensions.rs +++ b/crates/llm/src/extensions.rs @@ -1,6 +1,7 @@ use std::any::{Any, TypeId}; use std::collections::HashMap; use std::sync::Arc; + /// Type-erased side-channel carrying provider-specific options on an /// [`LlmRequest`](crate::LlmRequest). Keys are types, so a value's meaning is /// defined by whoever defines the key type. @@ -30,6 +31,14 @@ impl Extensions { .get(&TypeId::of::()) .and_then(|value| value.as_ref().downcast_ref::()) } + + /// Copy every entry from `other`, overwriting same-keyed values. Use to + /// layer per-request extensions (e.g. session id) over defaults held + /// elsewhere (e.g. provider options from settings). + pub fn merge(&mut self, other: &Extensions) { + self.map + .extend(other.map.iter().map(|(key, value)| (*key, value.clone()))); + } } /// Canonical session identity, set by the agent for every request. Codecs @@ -72,4 +81,21 @@ mod tests { assert_eq!(extensions.get::().unwrap().0, "abc"); assert_eq!(detached.get::().unwrap().0, "other"); } + + #[test] + fn merge_layers_other_over_self() { + struct Routing(Vec); + + let mut base = Extensions::default(); + base.insert(SessionId("abc".into())); + base.insert(Routing(vec!["a".into()])); + + let mut per_request = Extensions::default(); + per_request.insert(SessionId("xyz".into())); + + base.merge(&per_request); + + assert_eq!(base.get::().unwrap().0, "xyz"); + assert_eq!(base.get::().unwrap().0, vec!["a"]); + } } diff --git a/crates/llm/src/request.rs b/crates/llm/src/request.rs index 475f59a..599b9fa 100644 --- a/crates/llm/src/request.rs +++ b/crates/llm/src/request.rs @@ -99,10 +99,6 @@ impl PromptCacheControl { pub struct RequestOptions { pub temperature: Option, pub max_tokens: Option, - /// OpenRouter sticky-session key for keeping a conversation on one provider. - pub session_id: Option, - /// cache routing key for requests sharing a prefix. - pub prompt_cache_key: Option, /// Provider-translated cache breakpoint/TTL configuration. pub cache_control: Option, } @@ -111,6 +107,8 @@ pub struct CompletionInput<'a> { pub messages: &'a [Message], pub tools: &'a [ToolSpec], pub options: &'a RequestOptions, + /// Provider-specific options forwarded to codecs with the request. + pub extensions: &'a Extensions, } pub struct LlmRequest<'a> { @@ -120,9 +118,6 @@ pub struct LlmRequest<'a> { pub options: &'a RequestOptions, pub credential: Option<&'a Credential>, pub reasoning_effort: ReasoningEffort, - /// Ordered provider list forwarded to the API. `None` means the request - /// carries no `provider` block at all. - pub provider_order: Option<&'a [String]>, /// Provider-specific options, read by codecs when building the wire body. pub extensions: Extensions, } diff --git a/crates/llm/tests/interactions_live.rs b/crates/llm/tests/interactions_live.rs index a658770..580be35 100644 --- a/crates/llm/tests/interactions_live.rs +++ b/crates/llm/tests/interactions_live.rs @@ -41,7 +41,6 @@ async fn live_text_completion() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, extensions: llm::Extensions::default(), }; @@ -82,7 +81,6 @@ async fn live_function_call() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, extensions: llm::Extensions::default(), }; @@ -126,7 +124,6 @@ async fn live_function_call_round_trip() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, extensions: llm::Extensions::default(), }; let first_response = api.complete(first_request).await.unwrap(); @@ -166,7 +163,6 @@ async fn live_function_call_round_trip() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, extensions: llm::Extensions::default(), }; let second_response = api.complete(second_request).await.unwrap(); diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index fff7e37..f01a8cf 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -20,7 +20,7 @@ pub use credentials::{ pub use google::{GoogleBuilder, GoogleProvider}; pub use local::{LocalApi, LocalModelEntry, LocalProvider, bind_local_model}; pub use model::{Model, ModelError, ModelOptions}; -pub use openrouter::{OpenRouterBuilder, OpenRouterProvider}; +pub use openrouter::{OpenRouterBuilder, OpenRouterProvider, Options as OpenRouterOptions}; pub use provider::{Provider, ProviderError, ProviderRegistry, bind_model}; pub use store::LocalModelStore; pub use zai::{ZaiBuilder, ZaiProvider}; diff --git a/crates/providers/src/model.rs b/crates/providers/src/model.rs index 5f1c2d6..3960e45 100644 --- a/crates/providers/src/model.rs +++ b/crates/providers/src/model.rs @@ -19,10 +19,8 @@ pub enum ModelError { pub struct ModelOptions { pub server_tools: Vec, pub reasoning_effort: ReasoningEffort, - /// Ordered provider list forwarded to the API. Empty means the request - /// carries no `provider` block at all. - pub provider_order: Vec, - /// Provider-specific options forwarded to codecs with every request. + /// Provider-specific options (like OpenRouter routing) forwarded to + /// codecs with every request. pub extensions: llm::Extensions, } @@ -31,7 +29,6 @@ impl From<&Model> for ModelOptions { ModelOptions { server_tools: value.server_tools.clone(), reasoning_effort: value.reasoning_effort(), - provider_order: value.provider_order.clone(), extensions: value.extensions.clone(), } } @@ -44,7 +41,6 @@ pub struct Model { auth: Arc, server_tools: Vec, reasoning_effort: ReasoningEffort, - provider_order: Vec, extensions: Extensions, } @@ -61,7 +57,6 @@ impl Model { auth, server_tools: options.server_tools, reasoning_effort: options.reasoning_effort, - provider_order: options.provider_order, extensions: options.extensions, } } @@ -81,8 +76,8 @@ impl Model { self.reasoning_effort = reasoning_effort; } - pub fn set_provider_order(&mut self, order: Vec) { - self.provider_order = order; + pub fn set_extensions(&mut self, extensions: Extensions) { + self.extensions = extensions; } fn tools<'a>(&'a self, local: &'a [ToolSpec]) -> Vec { @@ -96,6 +91,8 @@ impl Model { pub async fn complete(&self, input: CompletionInput<'_>) -> Result { let credential = self.auth.resolve().await?; let tools = self.tools(input.tools); + let mut extensions = self.extensions.clone(); + extensions.merge(input.extensions); let request = LlmRequest { model_id: &self.info.id, messages: input.messages, @@ -103,9 +100,7 @@ impl Model { options: input.options, credential: Some(&credential), reasoning_effort: self.reasoning_effort, - provider_order: (!self.provider_order.is_empty()) - .then_some(self.provider_order.as_slice()), - extensions: self.extensions.clone(), + extensions, }; Ok(self.api.complete(request).await?) } @@ -113,6 +108,8 @@ impl Model { pub async fn stream(&self, input: CompletionInput<'_>) -> Result { let credential = self.auth.resolve().await?; let tools = self.tools(input.tools); + let mut extensions = self.extensions.clone(); + extensions.merge(input.extensions); let request = LlmRequest { model_id: &self.info.id, messages: input.messages, @@ -120,9 +117,7 @@ impl Model { options: input.options, credential: Some(&credential), reasoning_effort: self.reasoning_effort, - provider_order: (!self.provider_order.is_empty()) - .then_some(self.provider_order.as_slice()), - extensions: self.extensions.clone(), + extensions, }; Ok(self.api.stream(request).await?) } diff --git a/crates/providers/src/openrouter.rs b/crates/providers/src/openrouter.rs index e6964fa..0a179e1 100644 --- a/crates/providers/src/openrouter.rs +++ b/crates/providers/src/openrouter.rs @@ -131,7 +131,6 @@ mod codec_tests { options: &options, credential: None, reasoning_effort: llm::ReasoningEffort::None, - provider_order: None, extensions, }; let body = OpenRouterCodec.request(&request).unwrap(); From 078c4c3f5e4832a0447a39f2ece8914d7b548c3e Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 12:41:33 +0530 Subject: [PATCH 05/10] docs: changelog for provider codec extraction --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bcbfab..1b67b2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Provider-specific request/response handling moved out of the generic chat completions codec into per-provider codecs (`LlmApiCodec`). OpenRouter's provider routing, sticky session id, and usage cost parsing now live in `OpenRouterCodec`; other OpenAI-compatible providers no longer receive those OpenRouter-only body fields. Provider options travel via typed request extensions instead of fields on the shared request types. + ## 0.1.4 - 2026-10-4 ### Added From 0be3a677664c56b85709de65b6a79c5d087937a4 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 16:04:13 +0530 Subject: [PATCH 06/10] docs: simply doc comments --- CHANGELOG.md | 2 +- crates/llm/src/extensions.rs | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b67b2d..56cccf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Provider-specific request/response handling moved out of the generic chat completions codec into per-provider codecs (`LlmApiCodec`). OpenRouter's provider routing, sticky session id, and usage cost parsing now live in `OpenRouterCodec`; other OpenAI-compatible providers no longer receive those OpenRouter-only body fields. Provider options travel via typed request extensions instead of fields on the shared request types. +- Provider-specific request/response handling moved out of the generic chat completions codec into per-provider codecs. ## 0.1.4 - 2026-10-4 diff --git a/crates/llm/src/extensions.rs b/crates/llm/src/extensions.rs index bf20396..1642609 100644 --- a/crates/llm/src/extensions.rs +++ b/crates/llm/src/extensions.rs @@ -3,16 +3,7 @@ use std::collections::HashMap; use std::sync::Arc; /// Type-erased side-channel carrying provider-specific options on an -/// [`LlmRequest`](crate::LlmRequest). Keys are types, so a value's meaning is -/// defined by whoever defines the key type. -/// -/// Keys live next to their consumers: generic keys (like -/// [`SessionId`]) in `llm`, provider-specific ones (like OpenRouter routing -/// options) in `providers`. The container never inspects what it stores, so -/// the dependency direction `llm <- providers <- agent <- alan` is preserved. -/// -/// Cloning is cheap: values live behind an [`Arc`], so requests can carry -/// their own copy. Insertion is intended before a request is shared. +/// [`LlmRequest`](crate::LlmRequest). #[derive(Clone, Default)] pub struct Extensions { map: HashMap>, From 3c4e0d01ff595d003e53019c70803d78344bb71d Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 16:04:33 +0530 Subject: [PATCH 07/10] alan: remove google from providers --- crates/alan/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/alan/src/main.rs b/crates/alan/src/main.rs index 83a6ba4..ac5e4c0 100644 --- a/crates/alan/src/main.rs +++ b/crates/alan/src/main.rs @@ -22,7 +22,7 @@ use agent::{Agent, SessionManager, ToolPermissionManager, default_tools}; use alan_tui::Runtime; use llm::ReasoningEffort; use providers::{ - FileCredentialStore, GoogleProvider, LocalProvider, ModelOptions, OpenRouterProvider, Provider, + FileCredentialStore, LocalProvider, ModelOptions, OpenRouterProvider, Provider, ProviderRegistry, ZaiProvider, bind_model, }; use std::path::PathBuf; @@ -151,7 +151,8 @@ fn build_providers( ) -> anyhow::Result>> { Ok(vec![ Arc::new(ZaiProvider::from_store(Arc::clone(credential_store)).build()?), - Arc::new(GoogleProvider::from_store(Arc::clone(credential_store)).build()?), + // until there really exists some evidence that this is needed, we can add back + // Arc::new(GoogleProvider::from_store(Arc::clone(credential_store)).build()?), Arc::new(OpenRouterProvider::from_store(Arc::clone(credential_store)).build()?), Arc::clone(local_provider) as Arc, ]) From 132accb8d1f37550434fdd1fc5cfd361bcc6df05 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 16:12:52 +0530 Subject: [PATCH 08/10] refactor: move provider-specific wire types out of llm crate --- crates/llm/src/apis/chat_completions/codec.rs | 10 +++------- crates/llm/src/apis/chat_completions/mod.rs | 2 +- crates/providers/src/openrouter.rs | 9 ++++++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index 37c27f5..7065e36 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -80,13 +80,6 @@ pub fn decode_stream_response(response: &BaseResponse) -> Result, - pub allow_fallbacks: bool, -} - #[derive(Serialize)] pub struct WireMessage { pub role: &'static str, @@ -96,6 +89,9 @@ pub struct WireMessage { #[serde(skip_serializing_if = "Option::is_none")] pub tool_call_id: Option, #[serde(skip_serializing_if = "Option::is_none")] + /// Always `None`: reasoning history is never re-sent on the wire + /// (OpenAI-compatible APIs reject or mis-handle round-tripped + /// `reasoning_details`). pub reasoning_details: Option>, } diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index a939beb..3e81c95 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -3,7 +3,7 @@ mod codec; pub use codec::{ BaseRequest, BaseResponse, DefaultChatCompletionsCodec, StreamChoice, StreamDelta, StreamFunctionWire, StreamToolCallWire, WireDefinition, WireFunction, WireMessage, - WireProvider, WireReasoning, WireTool, WireToolCall, WireUsage, decode_stream_response, + WireReasoning, WireTool, WireToolCall, WireUsage, decode_stream_response, stop_reason_for_finish_reason, usage_from_wire, }; diff --git a/crates/providers/src/openrouter.rs b/crates/providers/src/openrouter.rs index 0a179e1..9451ffa 100644 --- a/crates/providers/src/openrouter.rs +++ b/crates/providers/src/openrouter.rs @@ -4,7 +4,7 @@ use crate::{ }; use async_trait::async_trait; use llm::apis::chat_completions::{ - BaseRequest, BaseResponse, WireProvider, WireUsage, decode_stream_response, + BaseRequest, BaseResponse, WireUsage, decode_stream_response, }; use llm::{ChatCompletionsApi, CodecChunk, HttpClient, LlmApi, LlmApiCodec, LlmError, LlmRequest}; use reqwest::StatusCode; @@ -24,6 +24,13 @@ pub struct Options { pub provider_order: Vec, } +/// OpenRouter's `provider` routing block: which upstream providers to prefer. +#[derive(Serialize)] +pub(crate) struct WireProvider { + pub(crate) only: Vec, + pub(crate) allow_fallbacks: bool, +} + #[derive(Serialize)] struct OpenRouterRequest<'a> { #[serde(flatten)] From 83d3e97766702c5702da33aa4d00a7a495f8505a Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 16:23:11 +0530 Subject: [PATCH 09/10] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cccf6..f507da1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Provider-specific request/response handling moved out of the generic chat completions codec into per-provider codecs. +- Remove google from active providers. ## 0.1.4 - 2026-10-4 From e9aa3671a8297923f075840ef7ee4af7fc995003 Mon Sep 17 00:00:00 2001 From: Revantark Date: Mon, 5 Oct 2026 16:24:10 +0530 Subject: [PATCH 10/10] fmt: openrouter --- crates/providers/src/openrouter.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/providers/src/openrouter.rs b/crates/providers/src/openrouter.rs index 9451ffa..a11372f 100644 --- a/crates/providers/src/openrouter.rs +++ b/crates/providers/src/openrouter.rs @@ -3,9 +3,7 @@ use crate::{ ProviderError, ProviderId, ServerToolInfo, }; use async_trait::async_trait; -use llm::apis::chat_completions::{ - BaseRequest, BaseResponse, WireUsage, decode_stream_response, -}; +use llm::apis::chat_completions::{BaseRequest, BaseResponse, WireUsage, decode_stream_response}; use llm::{ChatCompletionsApi, CodecChunk, HttpClient, LlmApi, LlmApiCodec, LlmError, LlmRequest}; use reqwest::StatusCode; use serde::{Deserialize, Serialize};