fix: keep an endpoint's catalog when its model listing fails

crates/douane/src/main.rs

@@ -64,6 +64,14 @@ async fn main() -> Result<()> {
             if pass.is_multiple_of(20) {
                 poller.quotas.save().await;
             }
+            // A live model listing that failed is read again every minute.
+            if pass.is_multiple_of(4)
+                && poller
+                    .models_incomplete
+                    .load(std::sync::atomic::Ordering::Relaxed)
+            {
+                poller.refresh_models().await;
+            }
             tokio::time::sleep(Duration::from_secs(15)).await;
         }
     });

crates/douane/src/state.rs

@@ -91,6 +91,9 @@ pub struct AppState {
     pub clients: Arc<crate::clients::Clients>,
     pub client: reqwest::Client,
     pub models: RwLock<Vec<Model>>,
+    /// Set when the last catalog refresh could not read an account's live
+    /// listing; the background loop retries until every listing answers.
+    pub models_incomplete: std::sync::atomic::AtomicBool,
     pub scheduler: Mutex<Scheduler>,
     pub affinity: Mutex<crate::affinity::Affinity>,
     pub(crate) antigravity_replay: Mutex<crate::antigravity_replay::ReplayCache>,
@@ -145,6 +148,7 @@ impl AppState {
             clients,
             client,
             models: RwLock::new(vec![]),
+            models_incomplete: std::sync::atomic::AtomicBool::new(false),
             scheduler: Mutex::new(Scheduler::default()),
             affinity: Mutex::new(crate::affinity::Affinity::default()),
             antigravity_replay: Mutex::new(crate::antigravity_replay::ReplayCache::default()),
@@ -188,6 +192,8 @@ impl AppState {
 
     pub async fn refresh_models(&self) {
         let bundled = catalog();
+        let previous = self.models.read().await.clone();
+        let mut incomplete = false;
         let mut models: BTreeMap<String, Model> = BTreeMap::new();
         let disabled = self.config.read().await.disabled_models.clone();
         for summary in self.accounts.list().await {
@@ -235,45 +241,77 @@ impl AppState {
                     } else {
                         format!("{root}/v1/models")
                     };
-                    if let Ok(response) = self
+                    let listing = self
                         .client
                         .get(url)
                         .bearer_auth(&account.access_token)
                         .timeout(std::time::Duration::from_secs(15))
                         .send()
                         .await
-                    {
-                        if response.status().is_success() {
-                            if let Ok(value) = response.json::<Value>().await {
-                                if let Some(data) = value["data"].as_array() {
-                                    available = data
-                                        .iter()
-                                        .filter_map(|m| {
-                                            Some(Model {
-                                                id: m["id"].as_str()?.into(),
-                                                name: m["name"]
-                                                    .as_str()
-                                                    .or_else(|| m["id"].as_str())?
-                                                    .into(),
-                                                provider: account.provider,
-                                                context_length: m["context_length"]
-                                                    .as_u64()
-                                                    .or_else(|| m["context_window"].as_u64()),
-                                                max_output_tokens: None,
-                                                enabled: true,
-                                                upstream_id: String::new(),
-                                                thinking: m.get("thinking").cloned(),
-                                                ..Model::default()
-                                            })
-                                        })
-                                        .filter(|m| {
-                                            account.models.is_empty()
-                                                || account.models.contains(&m.id)
-                                        })
-                                        .collect();
+                        .ok()
+                        .filter(|response| response.status().is_success());
+                    let data = match listing {
+                        Some(response) => response.json::<Value>().await.ok(),
+                        None => None,
+                    };
+                    if let Some(data) = data.as_ref().and_then(|v| v["data"].as_array()) {
+                        available = data
+                            .iter()
+                            .filter_map(|m| {
+                                Some(Model {
+                                    id: m["id"].as_str()?.into(),
+                                    name: m["name"].as_str().or_else(|| m["id"].as_str())?.into(),
+                                    provider: account.provider,
+                                    context_length: m["context_length"]
+                                        .as_u64()
+                                        .or_else(|| m["context_window"].as_u64()),
+                                    max_output_tokens: None,
+                                    enabled: true,
+                                    upstream_id: String::new(),
+                                    thinking: m.get("thinking").cloned(),
+                                    ..Model::default()
+                                })
+                            })
+                            .filter(|m| account.models.is_empty() || account.models.contains(&m.id))
+                            .collect();
+                    } else {
+                        // A listing that times out or errors once must not
+                        // empty the pool until the next restart: keep what
+                        // the last refresh read, within the account's own
+                        // model list. An account that names its models keeps
+                        // them without a listing, so only one that names
+                        // none is retried in the background.
+                        let pool = account.pool();
+                        tracing::warn!(pool = %pool, "model listing unavailable, keeping the previous catalog");
+                        let kept: Vec<Model> = previous
+                            .iter()
+                            .filter(|m| {
+                                m.provider == account.provider
+                                    && m.id
+                                        .strip_prefix(pool)
+                                        .is_some_and(|rest| rest.starts_with('/'))
+                                    && (account.models.is_empty()
+                                        || account.models.contains(&m.upstream_id))
+                            })
+                            .map(|m| Model {
+                                id: m.upstream_id.clone(),
+                                ..m.clone()
+                            })
+                            .collect();
+                        if account.models.is_empty() {
+                            available = kept;
+                        } else {
+                            // Every named model stays, with what the last
+                            // listing said about it when it said something.
+                            for model in &mut available {
+                                if let Some(known) =
+                                    kept.iter().find(|k| k.upstream_id == model.upstream_id)
+                                {
+                                    *model = known.clone();
                                 }
                             }
                         }
+                        incomplete |= account.models.is_empty();
                     }
                 }
             }
@@ -301,6 +339,8 @@ impl AppState {
             }
         }
         *self.models.write().await = models.into_values().collect();
+        self.models_incomplete
+            .store(incomplete, std::sync::atomic::Ordering::Relaxed);
     }
 
     /// Copilot's own catalog: the models this account's plan enables, with

crates/douane/tests/endpoints.rs

@@ -401,6 +401,114 @@ async fn an_endpoint_route_must_not_shadow_a_provider() {
     }
 }
 
+#[tokio::test]
+async fn a_failed_listing_keeps_the_previous_catalog() {
+    // OpenRouter's listing once timed out during a refresh and its whole
+    // pool vanished from /v1/models until douane restarted.
+    let failing = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let switch = failing.clone();
+    let server = serve(Router::new().route(
+        "/v1/models",
+        get(move || {
+            let failing = switch.load(std::sync::atomic::Ordering::Relaxed);
+            async move {
+                if failing {
+                    StatusCode::BAD_GATEWAY.into_response()
+                } else {
+                    Json(json!({"data":[{"id":"deepseek/deepseek-v4.1-flash"}]})).into_response()
+                }
+            }
+        }),
+    ))
+    .await;
+    let gateway = Gateway::new(&[endpoint("openrouter", &server.url, "key")]).await;
+    let listed = ["openrouter/deepseek/deepseek-v4.1-flash"];
+    assert_eq!(gateway.model_ids().await, listed);
+
+    failing.store(true, std::sync::atomic::Ordering::Relaxed);
+    gateway.state.refresh_models().await;
+    assert_eq!(gateway.model_ids().await, listed);
+    assert!(
+        gateway
+            .state
+            .models_incomplete
+            .load(std::sync::atomic::Ordering::Relaxed)
+    );
+
+    failing.store(false, std::sync::atomic::Ordering::Relaxed);
+    gateway.state.refresh_models().await;
+    assert_eq!(gateway.model_ids().await, listed);
+    assert!(
+        !gateway
+            .state
+            .models_incomplete
+            .load(std::sync::atomic::Ordering::Relaxed)
+    );
+}
+
+#[tokio::test]
+async fn an_endpoint_without_a_listing_keeps_its_named_models() {
+    // A request path that works beside a `/models` that never answers.
+    let server = serve(Router::new()).await;
+    let mut account = endpoint("mimo", &server.url, "key");
+    account["models"] = json!(["mimo-v3"]);
+    let gateway = Gateway::new(&[account]).await;
+    assert_eq!(gateway.model_ids().await, ["mimo/mimo-v3"]);
+    gateway.state.refresh_models().await;
+    assert_eq!(gateway.model_ids().await, ["mimo/mimo-v3"]);
+    assert!(
+        !gateway
+            .state
+            .models_incomplete
+            .load(std::sync::atomic::Ordering::Relaxed)
+    );
+}
+
+#[tokio::test]
+async fn a_narrowed_model_list_applies_while_the_listing_fails() {
+    let failing = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let switch = failing.clone();
+    let server = serve(Router::new().route(
+        "/v1/models",
+        get(move || {
+            let failing = switch.load(std::sync::atomic::Ordering::Relaxed);
+            async move {
+                if failing {
+                    StatusCode::BAD_GATEWAY.into_response()
+                } else {
+                    Json(json!({"data":[{"id":"a","context_length":64000},{"id":"b"}]}))
+                        .into_response()
+                }
+            }
+        }),
+    ))
+    .await;
+    let gateway = Gateway::new(&[endpoint("openrouter", &server.url, "key")]).await;
+    assert_eq!(gateway.model_ids().await, ["openrouter/a", "openrouter/b"]);
+    failing.store(true, std::sync::atomic::Ordering::Relaxed);
+    let id = gateway.state.accounts.list().await[0].id.clone();
+    gateway
+        .state
+        .accounts
+        .update_endpoint(
+            &id,
+            douane::subscriptions::EndpointPatch {
+                models: Some(vec!["a".into()]),
+                ..Default::default()
+            },
+        )
+        .await
+        .unwrap();
+    gateway.state.refresh_models().await;
+    assert_eq!(gateway.model_ids().await, ["openrouter/a"]);
+    // The management catalog drops `b` too, and what the last listing said
+    // about `a` survives the outage.
+    let models = gateway.state.models.read().await;
+    let ids: Vec<&str> = models.iter().map(|m| m.id.as_str()).collect();
+    assert_eq!(ids, ["openrouter/a"]);
+    assert_eq!(models[0].context_length, Some(64000));
+}
+
 #[tokio::test]
 async fn copilot_lists_enabled_models_and_uses_each_ones_path() {
     let (server, seen) = copilot().await;