diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bcbfab..f507da1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Provider-specific request/response handling moved out of the generic chat completions codec into per-provider codecs. +- Remove google from active providers. + ## 0.1.4 - 2026-10-4 ### Added diff --git a/crates/agent/src/agent/mod.rs b/crates/agent/src/agent/mod.rs index 48a2505..ba34321 100644 --- a/crates/agent/src/agent/mod.rs +++ b/crates/agent/src/agent/mod.rs @@ -370,13 +370,13 @@ impl Agent { Ok(()) } - pub async fn set_provider_order(&self, provider_order: Vec) -> Result<(), AgentError> { + pub async fn set_extensions(&self, extensions: llm::Extensions) -> Result<(), AgentError> { let mut model = self.model.try_lock().map_err(|_| { AgentError::Model(providers::ModelError::Llm(llm::LlmError::Configuration( - "agent is busy: cannot change provider order mid-run".into(), + "agent is busy: cannot change request extensions mid-run".into(), ))) })?; - model.set_provider_order(provider_order); + model.set_extensions(extensions); Ok(()) } } diff --git a/crates/agent/src/agent/prompt.rs b/crates/agent/src/agent/prompt.rs index 7751917..e30a573 100644 --- a/crates/agent/src/agent/prompt.rs +++ b/crates/agent/src/agent/prompt.rs @@ -184,11 +184,13 @@ pub(super) async fn summarize( messages.push(Message::user(instruction)); let options = RequestOptions::default(); + let extensions = llm::Extensions::default(); let mut stream_resp = model .stream(CompletionInput { messages: &messages, tools: &[], options: &options, + extensions: &extensions, }) .await?; @@ -217,9 +219,9 @@ pub(super) async fn stream_round( .collect(); let messages = build_messages(context); + let mut extensions = llm::Extensions::default(); + extensions.insert(llm::SessionId(session_id)); let options = RequestOptions { - prompt_cache_key: Some(session_id.clone()), - session_id: Some(session_id), cache_control: Some(PromptCacheControl::one_hour()), ..RequestOptions::default() }; @@ -229,6 +231,7 @@ pub(super) async fn stream_round( messages: &messages, tools: &tools, options: &options, + extensions: &extensions, }) .await?; diff --git a/crates/alan/src/core/settings.rs b/crates/alan/src/core/settings.rs index 04685d8..0cc4c8f 100644 --- a/crates/alan/src/core/settings.rs +++ b/crates/alan/src/core/settings.rs @@ -22,6 +22,7 @@ use crate::core::store::JsonStore; const SETTINGS_KEY: &str = "settings"; use anyhow::Context as _; use llm::ReasoningEffort; +use providers::OpenRouterOptions; use std::collections::BTreeMap; use std::path::PathBuf; @@ -146,6 +147,20 @@ impl Settings { pub fn provider_order(&self, model: &str) -> Vec { self.provider_orders.get(model).cloned().unwrap_or_default() } + + /// Provider-specific request options for `model`, keyed by provider. + /// Only OpenRouter exposes settings today; the order is routed to its + /// codec via request extensions. + pub fn extensions(&self, model: &str) -> llm::Extensions { + let mut extensions = llm::Extensions::default(); + let order = self.provider_order(model); + if !order.is_empty() { + extensions.insert(OpenRouterOptions { + provider_order: order, + }); + } + extensions + } } /// Apply a PatchSettings to mutate only specified fields. diff --git a/crates/alan/src/main.rs b/crates/alan/src/main.rs index ef44bd3..ac5e4c0 100644 --- a/crates/alan/src/main.rs +++ b/crates/alan/src/main.rs @@ -22,7 +22,7 @@ use agent::{Agent, SessionManager, ToolPermissionManager, default_tools}; use alan_tui::Runtime; use llm::ReasoningEffort; use providers::{ - FileCredentialStore, GoogleProvider, LocalProvider, ModelOptions, OpenRouterProvider, Provider, + FileCredentialStore, LocalProvider, ModelOptions, OpenRouterProvider, Provider, ProviderRegistry, ZaiProvider, bind_model, }; use std::path::PathBuf; @@ -151,7 +151,8 @@ fn build_providers( ) -> anyhow::Result>> { Ok(vec![ Arc::new(ZaiProvider::from_store(Arc::clone(credential_store)).build()?), - Arc::new(GoogleProvider::from_store(Arc::clone(credential_store)).build()?), + // until there really exists some evidence that this is needed, we can add back + // Arc::new(GoogleProvider::from_store(Arc::clone(credential_store)).build()?), Arc::new(OpenRouterProvider::from_store(Arc::clone(credential_store)).build()?), Arc::clone(local_provider) as Arc, ]) @@ -197,7 +198,7 @@ fn select_model( ModelOptions { server_tools, reasoning_effort: settings.reasoning.unwrap_or_default(), - provider_order: settings.provider_order(&model_id), + extensions: settings.extensions(&model_id), }, )?) } diff --git a/crates/alan/src/views/components/chat_view/models.rs b/crates/alan/src/views/components/chat_view/models.rs index fdba1a9..f049b0a 100644 --- a/crates/alan/src/views/components/chat_view/models.rs +++ b/crates/alan/src/views/components/chat_view/models.rs @@ -8,8 +8,8 @@ use crate::root::AlanAction; use alan_tui::TaskError; use alan_tui::context::Context; use providers::{ - Model, ModelInfo, ModelOptions, Provider, ProviderId, ProviderRegistry, bind_local_model, - bind_model, + Model, ModelInfo, ModelOptions, OpenRouterOptions, Provider, ProviderId, ProviderRegistry, + bind_local_model, bind_model, }; use std::sync::Arc; @@ -63,7 +63,7 @@ pub(crate) fn open_models_picker(view: &mut ChatView, cx: &mut Context<'_, ChatV let settings = settings::get_settings() .await .map_err(|e| TaskError(e.into()))?; - options.provider_order = settings.provider_order(&model_info.id); + options.extensions = settings.extensions(&model_info.id); let model = bind_model_info(&providers, &model_info, options)?; let name = model_info.name.clone(); @@ -124,8 +124,12 @@ pub(crate) fn apply_model_provider( cx.spawn( async move { let model_id = agent.info().await.id; + let mut extensions = agent.model_options().await.extensions; + extensions.insert(OpenRouterOptions { + provider_order: provider_order.clone(), + }); agent - .set_provider_order(provider_order.clone()) + .set_extensions(extensions) .await .map_err(|error| TaskError(Box::new(error)))?; diff --git a/crates/alan/src/views/components/chat_view/profiles.rs b/crates/alan/src/views/components/chat_view/profiles.rs index 4a9731e..e166d42 100644 --- a/crates/alan/src/views/components/chat_view/profiles.rs +++ b/crates/alan/src/views/components/chat_view/profiles.rs @@ -187,7 +187,7 @@ fn apply_profile( async move { let mut options = agent.model_options().await; let settings = settings::get_settings().await.map_err(super::to_task)?; - options.provider_order = settings.provider_order(&profile.model); + options.extensions = settings.extensions(&profile.model); options.reasoning_effort = profile.reasoning; options.server_tools = server_tools_for_provider( &providers, diff --git a/crates/llm/src/apis/chat_completions/codec.rs b/crates/llm/src/apis/chat_completions/codec.rs index 2603eaa..7065e36 100644 --- a/crates/llm/src/apis/chat_completions/codec.rs +++ b/crates/llm/src/apis/chat_completions/codec.rs @@ -1,200 +1,227 @@ +use crate::codec::{CodecChunk, LlmApiCodec}; use crate::{LlmError, LlmEvent, LlmRequest, Message, Role, StopReason, ToolSpec, Usage}; use serde::{Deserialize, Serialize}; -/// Providers to prioritize, in order. Sent only when the caller configures one; -/// otherwise the request carries no `provider` block at all. +/// Codec for the canonical OpenAI-compatible `/chat/completions` protocol. +pub struct DefaultChatCompletionsCodec; + +impl LlmApiCodec for DefaultChatCompletionsCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + serde_json::to_string(&BaseRequest::new(request)).map_err(LlmError::Serialization) + } + + fn response(&self, data: &str) -> Result { + let response: BaseResponse = serde_json::from_str(data).map_err(LlmError::Serialization)?; + decode_stream_response(&response) + } +} + +/// Canonical `/chat/completions` request body. Providers with extras embed it +/// via `#[serde(flatten)]` and add their own fields. #[derive(Serialize)] -struct Request<'a> { - model: &'a str, - stream: bool, - messages: Vec, +pub struct BaseRequest<'a> { + pub model: &'a str, + pub stream: bool, + pub messages: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] - tools: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - max_tokens: Option, - reasoning: WireReasoning, + pub tools: Vec, #[serde(skip_serializing_if = "Option::is_none")] - session_id: Option<&'a str>, + pub temperature: Option, #[serde(skip_serializing_if = "Option::is_none")] - prompt_cache_key: Option<&'a str>, + pub max_tokens: Option, + pub reasoning: WireReasoning, #[serde(skip_serializing_if = "Option::is_none")] - cache_control: Option<&'a crate::PromptCacheControl>, + pub prompt_cache_key: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] - provider: Option, + pub cache_control: Option<&'a crate::PromptCacheControl>, } -#[derive(Serialize)] -struct WireProvider { - only: Vec, - allow_fallbacks: bool, +impl<'a> BaseRequest<'a> { + pub fn new(request: &'a LlmRequest<'a>) -> Self { + Self { + model: request.model_id, + stream: true, + messages: request.messages.iter().map(wire_message).collect(), + tools: request.tools.iter().map(wire_tool).collect(), + temperature: request.options.temperature, + max_tokens: request.options.max_tokens, + reasoning: WireReasoning { + effort: request.reasoning_effort.to_string(), + }, + prompt_cache_key: request + .extensions + .get::() + .map(|session| session.0.as_str()), + cache_control: request.options.cache_control.as_ref(), + } + } +} + +/// Canonical `/chat/completions` stream chunk. Providers with extra fields +/// embed it via `#[serde(flatten)]` and patch the decoded result. +#[derive(Deserialize)] +pub struct BaseResponse { + pub model: Option, + pub choices: Vec, + pub usage: Option, +} + +/// Decode a canonical [`BaseResponse`] into the API-facing [`CodecChunk`]. +/// Exposed so provider codecs can reuse the canonical mapping after +/// flattening [`BaseResponse`] into their own wire chunk. +pub fn decode_stream_response(response: &BaseResponse) -> Result { + let chunk = stream_chunk_from_response(response)?; + let events = stream_events(&chunk); + Ok(CodecChunk { + model: chunk.model, + finish_reason: chunk.finish_reason, + usage: chunk.usage, + events, + }) } #[derive(Serialize)] -struct WireMessage { - role: &'static str, - content: Option, +pub struct WireMessage { + pub role: &'static str, + pub content: Option, #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, + pub tool_calls: Option>, #[serde(skip_serializing_if = "Option::is_none")] - tool_call_id: Option, + pub tool_call_id: Option, #[serde(skip_serializing_if = "Option::is_none")] - reasoning_details: Option>, + /// Always `None`: reasoning history is never re-sent on the wire + /// (OpenAI-compatible APIs reject or mis-handle round-tripped + /// `reasoning_details`). + pub reasoning_details: Option>, } #[derive(Serialize)] -struct WireToolCall { - id: String, +pub struct WireToolCall { + pub id: String, #[serde(rename = "type")] - kind: &'static str, - function: WireFunction, + pub kind: &'static str, + pub function: WireFunction, } #[derive(Serialize)] -struct WireFunction { - name: String, - arguments: String, +pub struct WireFunction { + pub name: String, + pub arguments: String, } #[derive(Serialize)] -struct WireTool { +pub struct WireTool { #[serde(rename = "type")] - kind: String, + pub kind: String, #[serde(skip_serializing_if = "Option::is_none")] - function: Option, + pub function: Option, } #[derive(Serialize)] -struct WireDefinition { - name: String, - description: String, - parameters: serde_json::Value, +pub struct WireDefinition { + pub name: String, + pub description: String, + pub parameters: serde_json::Value, } #[derive(Serialize)] -struct WireReasoning { - effort: String, +pub struct WireReasoning { + pub effort: String, } #[derive(Deserialize)] -struct StreamResponse { - model: Option, - choices: Vec, - usage: Option, +pub struct StreamChoice { + pub delta: Option, + pub finish_reason: Option, } #[derive(Deserialize)] -struct StreamChoice { - delta: Option, - finish_reason: Option, +pub struct StreamDelta { + pub content: Option, + pub tool_calls: Option>, + pub reasoning: Option, + pub reasoning_content: Option, + pub reasoning_details: Option>, } #[derive(Deserialize)] -struct StreamDelta { - content: Option, - tool_calls: Option>, - reasoning: Option, - reasoning_content: Option, - reasoning_details: Option>, +pub struct StreamToolCallWire { + pub index: Option, + pub id: Option, + pub function: Option, } #[derive(Deserialize)] -struct StreamToolCallWire { - index: Option, - id: Option, - function: Option, -} - -#[derive(Deserialize)] -struct StreamFunctionWire { - name: Option, - arguments: Option, +pub struct StreamFunctionWire { + pub name: Option, + pub arguments: Option, } +/// Canonical usage fields; the OpenAI-standard token accounting. #[derive(Deserialize, Default)] #[serde(default)] -struct WireUsage { - prompt_tokens: u64, - completion_tokens: u64, - total_tokens: Option, - cost: Option, - cost_details: Option, - prompt_tokens_details: Option, - completion_tokens_details: Option, -} - -#[derive(Deserialize, Clone, Default)] -#[serde(default)] -struct WireCostDetails { - upstream_inference_cost: Option, +pub struct WireUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: Option, + pub prompt_tokens_details: Option, + pub completion_tokens_details: Option, } #[derive(Deserialize, Clone, Default)] #[serde(default)] -struct WirePromptTokensDetails { - cached_tokens: Option, - cache_write_tokens: Option, - audio_tokens: Option, +pub struct WirePromptTokensDetails { + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub audio_tokens: Option, } #[derive(Deserialize, Clone, Default)] #[serde(default)] -struct WireCompletionTokensDetails { - reasoning_tokens: Option, +pub struct WireCompletionTokensDetails { + pub reasoning_tokens: Option, } -#[derive(Debug)] -pub(crate) struct StreamChunk { - pub(crate) model: Option, - pub(crate) text: Option, - pub(crate) reasoning: Option, - pub(crate) reasoning_details: Vec, - pub(crate) tool_calls: Vec, - pub(crate) finish_reason: Option, - pub(crate) usage: Option, -} - -#[derive(Debug)] -pub(crate) struct StreamToolCall { - pub(crate) index: usize, - pub(crate) id: Option, - pub(crate) name: Option, - pub(crate) arguments: String, +/// Map canonical [`WireUsage`] onto [`Usage`]. +pub fn usage_from_wire(wire: &WireUsage) -> Usage { + Usage { + input_tokens: wire.prompt_tokens, + output_tokens: wire.completion_tokens, + total_tokens: wire.total_tokens, + cost: None, + upstream_inference_cost: None, + cached_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.cached_tokens), + cache_write_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.cache_write_tokens), + reasoning_tokens: wire + .completion_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens), + audio_tokens: wire + .prompt_tokens_details + .as_ref() + .and_then(|d| d.audio_tokens), + } } -pub(crate) fn serialize_request(request: &LlmRequest<'_>) -> Result { - let messages = request.messages.iter().map(wire_message).collect(); - let tools = request.tools.iter().map(wire_tool).collect(); - let reasoning = WireReasoning { - effort: request.reasoning_effort.to_string(), - }; - let provider = request - .provider_order - .filter(|order| !order.is_empty()) - .map(|order| WireProvider { - only: order.to_vec(), - allow_fallbacks: order.is_empty(), - }); - - serde_json::to_string(&Request { - model: request.model_id, - stream: true, - messages, - tools, - temperature: request.options.temperature, - max_tokens: request.options.max_tokens, - reasoning, - session_id: request.options.session_id.as_deref(), - prompt_cache_key: request.options.prompt_cache_key.as_deref(), - cache_control: request.options.cache_control.as_ref(), - provider, +/// Map a canonical finish reason onto [`StopReason`]. +pub fn stop_reason_for_finish_reason(reason: Option<&str>) -> Option { + reason.map(|reason| match reason { + "tool_calls" | "function_call" => StopReason::ToolUse, + "length" => StopReason::Length, + "content_filter" => StopReason::ContentFilter, + "stop" => StopReason::Stop, + _ => StopReason::Error, }) - .map_err(LlmError::Serialization) } -pub(crate) fn deserialize_stream_chunk(body: &str) -> Result { - let response: StreamResponse = serde_json::from_str(body).map_err(LlmError::Serialization)?; +/// Decode a canonical [`BaseResponse`] into the internal stream chunk. +pub(crate) fn stream_chunk_from_response(response: &BaseResponse) -> Result { let choice = response.choices.first(); let delta = choice.and_then(|choice| choice.delta.as_ref()); let text = delta @@ -235,34 +262,39 @@ pub(crate) fn deserialize_stream_chunk(body: &str) -> Result Vec { +#[derive(Debug)] +pub(crate) struct StreamChunk { + pub(crate) model: Option, + pub(crate) text: Option, + pub(crate) reasoning: Option, + pub(crate) reasoning_details: Vec, + pub(crate) tool_calls: Vec, + pub(crate) finish_reason: Option, + pub(crate) usage: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct StreamToolCall { + pub(crate) index: usize, + pub(crate) id: Option, + pub(crate) name: Option, + pub(crate) arguments: String, +} + +pub(crate) fn stream_events(chunk: &StreamChunk) -> Vec { let mut events = Vec::new(); - let details = chunk.reasoning_details; - if let Some(reasoning) = chunk.reasoning { + let details = chunk.reasoning_details.clone(); + if let Some(reasoning) = chunk.reasoning.clone() { let details = if details.is_empty() { vec![serde_json::json!({ "type": "reasoning.text", @@ -278,12 +310,13 @@ pub(crate) fn stream_events(chunk: StreamChunk) -> Vec { events.push(LlmEvent::ReasoningDelta { reasoning, details }); } } - if let Some(text) = chunk.text { + if let Some(text) = chunk.text.clone() { events.push(LlmEvent::TextDelta { text }); } events.extend( chunk .tool_calls + .clone() .into_iter() .map(|call| LlmEvent::ToolCallDelta { index: call.index, @@ -308,16 +341,6 @@ fn reasoning_text(details: &[serde_json::Value]) -> String { .collect() } -pub(crate) fn stop_reason_for_finish_reason(reason: Option<&str>) -> Option { - reason.map(|reason| match reason { - "tool_calls" | "function_call" => StopReason::ToolUse, - "length" => StopReason::Length, - "content_filter" => StopReason::ContentFilter, - "stop" => StopReason::Stop, - _ => StopReason::Error, - }) -} - fn wire_message(message: &Message) -> WireMessage { let content = if let Some(parts) = &message.content_parts { Some(serde_json::to_value(parts).expect("content parts must serialize")) @@ -377,6 +400,10 @@ fn role(role: Role) -> &'static str { #[cfg(test)] mod tests { use super::*; + + fn codec() -> super::DefaultChatCompletionsCodec { + super::DefaultChatCompletionsCodec + } use crate::{RequestOptions, ToolDefinition, ToolSpec}; fn request<'a>( @@ -392,7 +419,7 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, + extensions: crate::Extensions::default(), } } @@ -409,7 +436,9 @@ mod tests { max_tokens: Some(128), ..RequestOptions::default() }; - let body = serialize_request(&request("model-a", &messages, &tools, &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &tools, &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(json["model"], "model-a"); @@ -437,7 +466,7 @@ mod tests { let mut request = request("model-a", &messages, &[], &options); request.reasoning_effort = crate::ReasoningEffort::High; let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); + serde_json::from_str(&codec().request(&request).unwrap()).unwrap(); assert_eq!(json["reasoning"]["effort"], "high"); assert!( json["messages"][0].get("reasoning_details").is_none(), @@ -453,7 +482,9 @@ mod tests { max_tokens: None, ..RequestOptions::default() }; - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); assert!(json.get("tools").is_none()); @@ -461,43 +492,6 @@ mod tests { assert!(json.get("max_tokens").is_none()); } - #[test] - fn omits_provider_block_when_unconfigured() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = None; - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert!(json.get("provider").is_none()); - } - - #[test] - fn omits_provider_block_when_order_is_empty() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = Some(&[]); - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert!(json.get("provider").is_none()); - } - - #[test] - fn serializes_provider_order_when_configured() { - let messages = [Message::user("hello")]; - let options = RequestOptions::default(); - let order = vec!["deepseek".to_string(), "fireworks".to_string()]; - let mut request = request("model-a", &messages, &[], &options); - request.provider_order = Some(&order); - let json: serde_json::Value = - serde_json::from_str(&serialize_request(&request).unwrap()).unwrap(); - assert_eq!( - json["provider"]["only"], - serde_json::json!(["deepseek", "fireworks"]) - ); - } - #[test] fn decodes_text_tool_calls_finish_reason_and_usage() { let body = r#"{ @@ -516,18 +510,30 @@ mod tests { "usage": {"prompt_tokens": 11, "completion_tokens": 7} }"#; - let chunk = deserialize_stream_chunk(body).unwrap(); + let chunk = codec().response(body).unwrap(); assert_eq!(chunk.model.as_deref(), Some("served-model")); - assert_eq!(chunk.text.as_deref(), Some("Checking")); - assert_eq!(chunk.tool_calls[0].name.as_deref(), Some("weather")); - assert_eq!(chunk.tool_calls[0].arguments, "{\"city\":"); + assert_eq!( + chunk.events, + vec![ + LlmEvent::TextDelta { + text: "Checking".into() + }, + LlmEvent::ToolCallDelta { + index: 0, + id: Some("call-1".into()), + name: Some("weather".into()), + arguments: "{\"city\":".into(), + signature: None, + }, + ] + ); assert_eq!(chunk.finish_reason.as_deref(), Some("tool_calls")); assert_eq!(chunk.usage.unwrap().output_tokens, 7); } #[test] fn rejects_tool_call_without_index() { - let error = deserialize_stream_chunk( + let error = codec().response( r#"{"choices":[{"delta":{"tool_calls":[{"id":"call-1","function":{"name":"bash"}}]}}]}"#, ) .unwrap_err(); @@ -536,11 +542,10 @@ mod tests { #[test] fn accepts_usage_only_chunk() { - let chunk = deserialize_stream_chunk( - r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#, - ) - .unwrap(); - assert!(chunk.text.is_none()); + let chunk = codec() + .response(r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#) + .unwrap(); + assert!(chunk.events.is_empty()); assert_eq!(chunk.usage.unwrap().input_tokens, 2); } @@ -571,7 +576,9 @@ mod tests { }, ])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; @@ -603,7 +610,9 @@ mod tests { }, ])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; @@ -619,7 +628,9 @@ mod tests { text: "hello".into(), }])]; let options = RequestOptions::default(); - let body = serialize_request(&request("model-a", &messages, &[], &options)).unwrap(); + let body = codec() + .request(&request("model-a", &messages, &[], &options)) + .unwrap(); let json: serde_json::Value = serde_json::from_str(&body).unwrap(); let content = &json["messages"][0]["content"]; diff --git a/crates/llm/src/apis/chat_completions/mod.rs b/crates/llm/src/apis/chat_completions/mod.rs index f804c14..3e81c95 100644 --- a/crates/llm/src/apis/chat_completions/mod.rs +++ b/crates/llm/src/apis/chat_completions/mod.rs @@ -1,6 +1,14 @@ mod codec; +pub use codec::{ + BaseRequest, BaseResponse, DefaultChatCompletionsCodec, StreamChoice, StreamDelta, + StreamFunctionWire, StreamToolCallWire, WireDefinition, WireFunction, WireMessage, + WireReasoning, WireTool, WireToolCall, WireUsage, decode_stream_response, + stop_reason_for_finish_reason, usage_from_wire, +}; + use crate::apis::sse::SseDecoder; +use crate::codec::{CodecChunk, LlmApiCodec}; use crate::{HttpClient, LlmApi, LlmError, LlmEvent, LlmRequest, LlmStream, StopReason}; use async_trait::async_trait; use futures_util::{StreamExt, stream}; @@ -12,6 +20,7 @@ const ENDPOINT: &str = "/chat/completions"; pub struct ChatCompletionsApi { base_url: String, http: Arc, + codec: Arc, } impl ChatCompletionsApi { @@ -19,14 +28,22 @@ impl ChatCompletionsApi { Self { base_url: base_url.into(), http, + codec: Arc::new(codec::DefaultChatCompletionsCodec), } } + + /// Attach a provider codec that translates between canonical requests + /// and this provider's wire format. + pub fn with_codec(mut self, codec: Arc) -> Self { + self.codec = codec; + self + } } #[async_trait] impl LlmApi for ChatCompletionsApi { async fn stream(&self, request: LlmRequest<'_>) -> Result { - let body = codec::serialize_request(&request)?; + let body = self.codec.request(&request)?; let url = format!("{}{}", self.base_url.trim_end_matches('/'), ENDPOINT); let response = self.http.post(&url, &body, request.credential).await?; let status = response.status(); @@ -38,7 +55,7 @@ impl LlmApi for ChatCompletionsApi { }); } - let state = StreamState::new(response.bytes_stream()); + let state = StreamState::new(response.bytes_stream(), self.codec.clone()); let output = stream::try_unfold(state, |mut state| async move { let event = state.next_event().await?; Ok::<_, LlmError>(event.map(|event| (event, state))) @@ -51,6 +68,7 @@ impl LlmApi for ChatCompletionsApi { struct StreamState { input: S, decoder: SseDecoder, + codec: Arc, pending: VecDeque, model: Option, stop_reason: Option, @@ -62,10 +80,11 @@ impl StreamState where S: futures_util::Stream> + Unpin, { - fn new(input: S) -> Self { + fn new(input: S, codec: Arc) -> Self { Self { input, decoder: SseDecoder::default(), + codec, pending: VecDeque::new(), model: None, stop_reason: None, @@ -115,7 +134,7 @@ where continue; } - let chunk = codec::deserialize_stream_chunk(&payload)?; + let chunk: CodecChunk = self.codec.response(&payload)?; if let Some(model) = chunk.model.clone() { self.model = Some(model); } @@ -130,7 +149,7 @@ where // Emit usage *after* the chunk's text/reasoning/tool-call deltas // so consumers see content before the summary. let usage_event = chunk.usage.clone().map(|usage| LlmEvent::Usage { usage }); - self.pending.extend(codec::stream_events(chunk)); + self.pending.extend(chunk.events); if let Some(event) = usage_event { self.pending.push_back(event); } @@ -142,6 +161,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::codec::CodecChunk; use crate::{Message, RequestOptions, ToolSpec}; use futures_util::StreamExt; use std::sync::Arc; @@ -161,7 +181,7 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, + extensions: crate::Extensions::default(), } } @@ -212,4 +232,80 @@ mod tests { assert_eq!(response.usage.unwrap().output_tokens, 3); assert!(server.await.unwrap().contains("\"stream\":true")); } + + /// A custom codec replaces both directions: the body it serializes is + /// sent, and the chunk it decodes drives the emitted events. + #[tokio::test] + async fn with_codec_uses_the_custom_codec() { + struct UppercaseCodec; + + impl LlmApiCodec for UppercaseCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + Ok(format!( + r#"{{"model":"{}","upper":true}}"#, + request.model_id + )) + } + + fn response(&self, _data: &str) -> Result { + Ok(CodecChunk { + model: Some("rewired".into()), + finish_reason: Some("stop".into()), + usage: None, + events: vec![LlmEvent::TextDelta { + text: "decoded by custom codec".into(), + }], + }) + } + } + + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 4096]; + loop { + let read = socket.read(&mut buffer).await.unwrap(); + assert!(read > 0); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\ndata: anything\n\ndata: [DONE]\n\n", + ) + .await + .unwrap(); + String::from_utf8(request).unwrap() + }); + + let api = + ChatCompletionsApi::new(format!("http://{address}/v1/"), Arc::new(HttpClient::new())) + .with_codec(Arc::new(UppercaseCodec)); + let messages = [Message::user("ping")]; + let options = RequestOptions::default(); + let mut events = api + .stream(request("model", &messages, &[], &options)) + .await + .unwrap(); + + let mut texts = Vec::new(); + let mut model = None; + let mut done = None; + while let Some(event) = events.next().await { + match event.unwrap() { + LlmEvent::TextDelta { text } => texts.push(text), + LlmEvent::Done { model: served, .. } => model = served, + other => done = Some(format!("{other:?}")), + } + } + + assert_eq!(texts.join(""), "decoded by custom codec"); + assert_eq!(model.as_deref(), Some("rewired")); + assert_eq!(done, None); + assert!(server.await.unwrap().contains(r#""upper":true"#)); + } } diff --git a/crates/llm/src/apis/interactions/codec.rs b/crates/llm/src/apis/interactions/codec.rs index a4a8ee0..20723b2 100644 --- a/crates/llm/src/apis/interactions/codec.rs +++ b/crates/llm/src/apis/interactions/codec.rs @@ -316,7 +316,7 @@ mod tests { options, credential: None, reasoning_effort: ReasoningEffort::None, - provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/apis/interactions/mod.rs b/crates/llm/src/apis/interactions/mod.rs index e03d9f5..93533f4 100644 --- a/crates/llm/src/apis/interactions/mod.rs +++ b/crates/llm/src/apis/interactions/mod.rs @@ -389,7 +389,7 @@ mod tests { options, credential: None, reasoning_effort: crate::ReasoningEffort::None, - provider_order: None, + extensions: crate::Extensions::default(), } } diff --git a/crates/llm/src/codec.rs b/crates/llm/src/codec.rs new file mode 100644 index 0000000..e1236b8 --- /dev/null +++ b/crates/llm/src/codec.rs @@ -0,0 +1,30 @@ +use crate::{LlmError, LlmEvent, LlmRequest, Usage}; + +/// One decoded SSE data frame from a provider stream. +#[derive(Debug)] +pub struct CodecChunk { + pub model: Option, + pub finish_reason: Option, + pub usage: Option, + /// Deltas carried by this frame, ready to emit in order. + pub events: Vec, +} + +/// Translates between canonical [`LlmRequest`]s / [`LlmEvent`]s and a +/// provider's wire format. +/// +/// An API's default codec speaks the canonical protocol shape. Providers with +/// extras implement this trait and attach their codec to the API (see +/// [`ChatCompletionsApi::with_codec`](crate::ChatCompletionsApi::with_codec)); +/// provider-specific request options arrive inside +/// [`LlmRequest::extensions`](crate::LlmRequest::extensions). +/// +/// Wire request/response structs stay private to each codec: the API only +/// needs a serialized body and decoded chunks. +pub trait LlmApiCodec: Send + Sync { + /// Serialize the wire body for `request`. + fn request(&self, request: &LlmRequest<'_>) -> Result; + + /// Decode one SSE `data:` frame from the provider stream. + fn response(&self, data: &str) -> Result; +} diff --git a/crates/llm/src/extensions.rs b/crates/llm/src/extensions.rs new file mode 100644 index 0000000..1642609 --- /dev/null +++ b/crates/llm/src/extensions.rs @@ -0,0 +1,92 @@ +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use std::sync::Arc; + +/// Type-erased side-channel carrying provider-specific options on an +/// [`LlmRequest`](crate::LlmRequest). +#[derive(Clone, Default)] +pub struct Extensions { + map: HashMap>, +} + +impl Extensions { + /// Store `value` under its type, replacing any previous value of the same + /// type. + pub fn insert(&mut self, value: T) { + self.map.insert(TypeId::of::(), Arc::new(value)); + } + + /// Borrow the value stored under type `T`, if any. + pub fn get(&self) -> Option<&T> { + self.map + .get(&TypeId::of::()) + .and_then(|value| value.as_ref().downcast_ref::()) + } + + /// Copy every entry from `other`, overwriting same-keyed values. Use to + /// layer per-request extensions (e.g. session id) over defaults held + /// elsewhere (e.g. provider options from settings). + pub fn merge(&mut self, other: &Extensions) { + self.map + .extend(other.map.iter().map(|(key, value)| (*key, value.clone()))); + } +} + +/// Canonical session identity, set by the agent for every request. Codecs +/// translate it per provider: OpenRouter sends it as `session_id` and +/// `prompt_cache_key`, vanilla OpenAI-compatible APIs as `prompt_cache_key`. +pub struct SessionId(pub String); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stores_and_returns_typed_values() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("abc".into())); + + assert_eq!(extensions.get::().unwrap().0, "abc"); + assert!(extensions.get::().is_none()); + } + + #[test] + fn replacing_a_key_overwrites_the_value() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("one".into())); + extensions.insert(SessionId("two".into())); + + assert_eq!(extensions.get::().unwrap().0, "two"); + } + + #[test] + fn clones_share_values_without_observing_each_other() { + let mut extensions = Extensions::default(); + extensions.insert(SessionId("abc".into())); + let clone = extensions.clone(); + + assert_eq!(clone.get::().unwrap().0, "abc"); + + let mut detached = extensions.clone(); + detached.insert(SessionId("other".into())); + assert_eq!(extensions.get::().unwrap().0, "abc"); + assert_eq!(detached.get::().unwrap().0, "other"); + } + + #[test] + fn merge_layers_other_over_self() { + struct Routing(Vec); + + let mut base = Extensions::default(); + base.insert(SessionId("abc".into())); + base.insert(Routing(vec!["a".into()])); + + let mut per_request = Extensions::default(); + per_request.insert(SessionId("xyz".into())); + + base.merge(&per_request); + + assert_eq!(base.get::().unwrap().0, "xyz"); + assert_eq!(base.get::().unwrap().0, vec!["a"]); + } +} diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index b75633a..6e80962 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -1,8 +1,10 @@ //! Provider-independent LLM protocol types and API implementations. mod api; +mod codec; mod error; mod event; +mod extensions; mod message; mod request; mod response; @@ -13,8 +15,10 @@ pub mod apis; pub use api::LlmApi; pub use apis::{ChatCompletionsApi, InteractionsApi}; +pub use codec::{CodecChunk, LlmApiCodec}; pub use error::LlmError; pub use event::{LlmEvent, LlmStream}; +pub use extensions::{Extensions, SessionId}; pub use message::{ContentPart, ImageUrl, Message, Role}; pub use request::{ CompletionInput, Credential, LlmRequest, PromptCacheControl, PromptCacheControlType, diff --git a/crates/llm/src/request.rs b/crates/llm/src/request.rs index d2ca4b1..599b9fa 100644 --- a/crates/llm/src/request.rs +++ b/crates/llm/src/request.rs @@ -1,4 +1,4 @@ -use crate::{Message, ToolSpec}; +use crate::{Extensions, Message, ToolSpec}; use serde::{Deserialize, Serialize}; use std::fmt; @@ -99,10 +99,6 @@ impl PromptCacheControl { pub struct RequestOptions { pub temperature: Option, pub max_tokens: Option, - /// OpenRouter sticky-session key for keeping a conversation on one provider. - pub session_id: Option, - /// cache routing key for requests sharing a prefix. - pub prompt_cache_key: Option, /// Provider-translated cache breakpoint/TTL configuration. pub cache_control: Option, } @@ -111,6 +107,8 @@ pub struct CompletionInput<'a> { pub messages: &'a [Message], pub tools: &'a [ToolSpec], pub options: &'a RequestOptions, + /// Provider-specific options forwarded to codecs with the request. + pub extensions: &'a Extensions, } pub struct LlmRequest<'a> { @@ -120,7 +118,6 @@ pub struct LlmRequest<'a> { pub options: &'a RequestOptions, pub credential: Option<&'a Credential>, pub reasoning_effort: ReasoningEffort, - /// Ordered provider list forwarded to the API. `None` means the request - /// carries no `provider` block at all. - pub provider_order: Option<&'a [String]>, + /// Provider-specific options, read by codecs when building the wire body. + pub extensions: Extensions, } diff --git a/crates/llm/tests/interactions_live.rs b/crates/llm/tests/interactions_live.rs index 99196fd..580be35 100644 --- a/crates/llm/tests/interactions_live.rs +++ b/crates/llm/tests/interactions_live.rs @@ -41,7 +41,7 @@ async fn live_text_completion() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -81,7 +81,7 @@ async fn live_function_call() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, + extensions: llm::Extensions::default(), }; let response = api.complete(request).await.unwrap(); @@ -124,7 +124,7 @@ async fn live_function_call_round_trip() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, + extensions: llm::Extensions::default(), }; let first_response = api.complete(first_request).await.unwrap(); let calls: Vec<_> = first_response.tool_calls().cloned().collect(); @@ -163,7 +163,7 @@ async fn live_function_call_round_trip() { options: &options, credential: Some(&credential), reasoning_effort: ReasoningEffort::None, - provider_order: None, + extensions: llm::Extensions::default(), }; let second_response = api.complete(second_request).await.unwrap(); assert!( diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index fff7e37..f01a8cf 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -20,7 +20,7 @@ pub use credentials::{ pub use google::{GoogleBuilder, GoogleProvider}; pub use local::{LocalApi, LocalModelEntry, LocalProvider, bind_local_model}; pub use model::{Model, ModelError, ModelOptions}; -pub use openrouter::{OpenRouterBuilder, OpenRouterProvider}; +pub use openrouter::{OpenRouterBuilder, OpenRouterProvider, Options as OpenRouterOptions}; pub use provider::{Provider, ProviderError, ProviderRegistry, bind_model}; pub use store::LocalModelStore; pub use zai::{ZaiBuilder, ZaiProvider}; diff --git a/crates/providers/src/model.rs b/crates/providers/src/model.rs index f6b8793..3960e45 100644 --- a/crates/providers/src/model.rs +++ b/crates/providers/src/model.rs @@ -1,8 +1,8 @@ use crate::auth::{AuthError, AuthResolver}; use crate::catalog::ModelInfo; use llm::{ - CompletionInput, LlmApi, LlmError, LlmRequest, LlmResponse, LlmStream, ReasoningEffort, - ServerTool, ToolSpec, + CompletionInput, Extensions, LlmApi, LlmError, LlmRequest, LlmResponse, LlmStream, + ReasoningEffort, ServerTool, ToolSpec, }; use std::sync::Arc; use thiserror::Error; @@ -19,9 +19,9 @@ pub enum ModelError { pub struct ModelOptions { pub server_tools: Vec, pub reasoning_effort: ReasoningEffort, - /// Ordered provider list forwarded to the API. Empty means the request - /// carries no `provider` block at all. - pub provider_order: Vec, + /// Provider-specific options (like OpenRouter routing) forwarded to + /// codecs with every request. + pub extensions: llm::Extensions, } impl From<&Model> for ModelOptions { @@ -29,7 +29,7 @@ impl From<&Model> for ModelOptions { ModelOptions { server_tools: value.server_tools.clone(), reasoning_effort: value.reasoning_effort(), - provider_order: value.provider_order.clone(), + extensions: value.extensions.clone(), } } } @@ -41,7 +41,7 @@ pub struct Model { auth: Arc, server_tools: Vec, reasoning_effort: ReasoningEffort, - provider_order: Vec, + extensions: Extensions, } impl Model { @@ -57,7 +57,7 @@ impl Model { auth, server_tools: options.server_tools, reasoning_effort: options.reasoning_effort, - provider_order: options.provider_order, + extensions: options.extensions, } } @@ -76,8 +76,8 @@ impl Model { self.reasoning_effort = reasoning_effort; } - pub fn set_provider_order(&mut self, order: Vec) { - self.provider_order = order; + pub fn set_extensions(&mut self, extensions: Extensions) { + self.extensions = extensions; } fn tools<'a>(&'a self, local: &'a [ToolSpec]) -> Vec { @@ -91,6 +91,8 @@ impl Model { pub async fn complete(&self, input: CompletionInput<'_>) -> Result { let credential = self.auth.resolve().await?; let tools = self.tools(input.tools); + let mut extensions = self.extensions.clone(); + extensions.merge(input.extensions); let request = LlmRequest { model_id: &self.info.id, messages: input.messages, @@ -98,8 +100,7 @@ impl Model { options: input.options, credential: Some(&credential), reasoning_effort: self.reasoning_effort, - provider_order: (!self.provider_order.is_empty()) - .then_some(self.provider_order.as_slice()), + extensions, }; Ok(self.api.complete(request).await?) } @@ -107,6 +108,8 @@ impl Model { pub async fn stream(&self, input: CompletionInput<'_>) -> Result { let credential = self.auth.resolve().await?; let tools = self.tools(input.tools); + let mut extensions = self.extensions.clone(); + extensions.merge(input.extensions); let request = LlmRequest { model_id: &self.info.id, messages: input.messages, @@ -114,8 +117,7 @@ impl Model { options: input.options, credential: Some(&credential), reasoning_effort: self.reasoning_effort, - provider_order: (!self.provider_order.is_empty()) - .then_some(self.provider_order.as_slice()), + extensions, }; Ok(self.api.stream(request).await?) } diff --git a/crates/providers/src/openrouter.rs b/crates/providers/src/openrouter.rs index 47cb2b1..a11372f 100644 --- a/crates/providers/src/openrouter.rs +++ b/crates/providers/src/openrouter.rs @@ -3,13 +3,195 @@ use crate::{ ProviderError, ProviderId, ServerToolInfo, }; use async_trait::async_trait; -use llm::{ChatCompletionsApi, HttpClient, LlmApi}; +use llm::apis::chat_completions::{BaseRequest, BaseResponse, WireUsage, decode_stream_response}; +use llm::{ChatCompletionsApi, CodecChunk, HttpClient, LlmApi, LlmApiCodec, LlmError, LlmRequest}; use reqwest::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; const BASE_URL: &str = "https://openrouter.ai/api/v1"; +/// OpenRouter's `/chat/completions` wire format: the canonical body plus +/// provider routing and a sticky-session key. +struct OpenRouterCodec; + +/// OpenRouter routing options, attached to a request's extensions by the +/// application. +pub struct Options { + /// Providers to prioritize, in order. + pub provider_order: Vec, +} + +/// OpenRouter's `provider` routing block: which upstream providers to prefer. +#[derive(Serialize)] +pub(crate) struct WireProvider { + pub(crate) only: Vec, + pub(crate) allow_fallbacks: bool, +} + +#[derive(Serialize)] +struct OpenRouterRequest<'a> { + #[serde(flatten)] + base: BaseRequest<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + session_id: Option<&'a str>, +} + +#[derive(Deserialize)] +struct OpenRouterStreamChunk { + model: Option, + choices: Vec, + usage: Option, +} + +/// OpenRouter extends the standard usage object with cost fields. +#[derive(Deserialize)] +struct OpenRouterUsage { + #[serde(flatten)] + base: WireUsage, + cost: Option, + cost_details: Option, +} + +#[derive(Deserialize)] +struct CostDetails { + upstream_inference_cost: Option, +} + +impl LlmApiCodec for OpenRouterCodec { + fn request(&self, request: &LlmRequest<'_>) -> Result { + let provider = request + .extensions + .get::() + .map(|options| options.provider_order.as_slice()) + .filter(|order| !order.is_empty()) + .map(|order| WireProvider { + only: order.to_vec(), + allow_fallbacks: order.is_empty(), + }); + let session_id = request.extensions.get::(); + let wire = OpenRouterRequest { + base: BaseRequest::new(request), + provider, + session_id: session_id.map(|session| session.0.as_str()), + }; + serde_json::to_string(&wire).map_err(LlmError::Serialization) + } + + fn response(&self, data: &str) -> Result { + let chunk: OpenRouterStreamChunk = + serde_json::from_str(data).map_err(LlmError::Serialization)?; + let OpenRouterUsage { + base, + cost, + cost_details, + } = match chunk.usage { + Some(usage) => usage, + None => { + return decode_stream_response(&BaseResponse { + model: chunk.model, + choices: chunk.choices, + usage: None, + }); + } + }; + let mut decoded = decode_stream_response(&BaseResponse { + model: chunk.model, + choices: chunk.choices, + usage: Some(base), + })?; + if let Some(usage) = &mut decoded.usage { + usage.cost = cost; + usage.upstream_inference_cost = cost_details + .as_ref() + .and_then(|details| details.upstream_inference_cost); + } + Ok(decoded) + } +} + +#[cfg(test)] +mod codec_tests { + use super::*; + use llm::{Extensions, Message, RequestOptions}; + + fn wire_request(order: Option>, session: Option<&str>) -> (String, Vec) { + let mut extensions = Extensions::default(); + if let Some(order) = order { + extensions.insert(Options { + provider_order: order, + }); + } + if let Some(session) = session { + extensions.insert(llm::SessionId(session.to_string())); + } + let messages = vec![Message::user("hello")]; + let options = RequestOptions::default(); + let request = LlmRequest { + model_id: "model-a", + messages: &messages, + tools: &[], + options: &options, + credential: None, + reasoning_effort: llm::ReasoningEffort::None, + extensions, + }; + let body = OpenRouterCodec.request(&request).unwrap(); + (body, messages) + } + + #[test] + fn serializes_provider_order_and_session_id() { + let (body, _keep) = wire_request( + Some(vec!["deepseek".into(), "fireworks".into()]), + Some("s-1"), + ); + let json: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + json["provider"]["only"], + serde_json::json!(["deepseek", "fireworks"]) + ); + assert_eq!(json["session_id"], "s-1"); + assert_eq!(json["model"], "model-a"); + } + + #[test] + fn omits_provider_and_session_when_unconfigured() { + let (body, _keep) = wire_request(None, None); + let json: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(json.get("provider").is_none()); + assert!(json.get("session_id").is_none()); + } + + #[test] + fn fills_cost_from_response() { + let chunk = OpenRouterCodec + .response( + r#"{"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":7,"cost":0.0021,"cost_details":{"upstream_inference_cost":0.0019}}}"#, + ) + .unwrap(); + let usage = chunk.usage.unwrap(); + assert_eq!(usage.input_tokens, 11); + assert_eq!(usage.output_tokens, 7); + assert_eq!(usage.cost, Some(0.0021)); + assert_eq!(usage.upstream_inference_cost, Some(0.0019)); + } + + #[test] + fn leaves_cost_unset_when_response_has_none() { + let chunk = OpenRouterCodec + .response(r#"{"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}}"#) + .unwrap(); + let usage = chunk.usage.unwrap(); + assert_eq!(usage.cost, None); + assert_eq!(usage.upstream_inference_cost, None); + } +} + pub struct OpenRouterProvider { id: ProviderId, models: std::sync::RwLock>, @@ -162,10 +344,10 @@ impl OpenRouterBuilder { pub fn build(self) -> Result { let api = self.api.unwrap_or_else(|| { - Arc::new(ChatCompletionsApi::new( - BASE_URL, - Arc::new(HttpClient::new()), - )) + Arc::new( + ChatCompletionsApi::new(BASE_URL, Arc::new(HttpClient::new())) + .with_codec(Arc::new(OpenRouterCodec)), + ) }); let auth = self .auth