fix(gateway): keep Realm-core authority during outages

CONTEXT.md

@@ -530,6 +530,8 @@ Status), in transit between two places (a pending Transfer, or a Shard's own row
 with no entity there), or offline. A negative Whereabouts, offline or no Character found at all,
 needs every configured World Shard to vouch that none of them is hiding the Character. When one
 cannot, the Whereabouts is unknown: another Shard's health, not a Transport Loss.
+An unavailable Realm-core also leaves negative Whereabouts unknown because the Gateway cannot
+exclude a pending Transfer.
 _Avoid_: presence state, location status
 
 **Away Status**:

docs/architecture.md

@@ -98,6 +98,11 @@ routing fact; module game logic never reads one, and an architecture test fails
   `gateway/src/stdb/store/<family>.rs` calls the reducer directly. When the call fails,
   `classify` in `gateway/src/stdb/connection.rs` decides. A Refusal gets the family's answer and
   the session continues. Every other failure is a Transport Loss and ends the World Session.
+- **Realm-core routing preserves authority.** `ShardRoutingStore::realm_store` returns the local
+  case only for a single-Shard Realm. An unavailable configured Realm-core returns an infrastructure
+  failure, so Mail, party and Loot Roll operations cannot fall back to a World Shard. Cleanup,
+  Companion Orders and Transfer use the same routing contract. A Realm Presence read that cannot
+  exclude a pending Transfer reports unknown Whereabouts.
 
 ### 2.3 The honest exception: gates that live in the gateway
 
@@ -622,6 +627,11 @@ Afterwards the gateway re-pins the session: new home handle, `bind_shard_session
 
 ### 6.3b Mail attachments across the boundary
 
+Mail recovery at world entry and after quest completion is best effort. If Realm-core is unavailable,
+recovery logs the failure and leaves Escrow pending for the next recovery attempt. Explicit Mail
+requests still end the World Session on Transport Loss. Recovery never commits Mail to another
+Shard because its authority is unavailable (`gateway/src/world/mail.rs`).
+
 The same problem with different nouns: a mail row is authoritative on realm-core, the sender's purse
 and items are on their own shard, and no transaction spans the two. `module/src/mail_escrow.rs`
 answers it with the same escrow shape — `realm_mail_fence` (the value leaves the purse INTO the

gateway/src/stdb/connection.rs

@@ -1248,7 +1248,7 @@ const LOOT_ROLL_RELAY_POLL: Duration = Duration::from_millis(200);
 
 /// Background loop driving `world::loot::relay_tick`: promotes each world shard's freshly
 /// staged loot rolls onto realm-core, and settles resolved winners back down. A no-op on an unsharded
-/// gateway — `relay_tick` returns immediately when `Coordinator::realm_store()` answers `None` — so
+/// gateway — `relay_tick` returns immediately when `Coordinator::realm_store()` answers `Ok(None)` — so
 /// this thread costs a single-database deployment nothing but its own idle wakeups.
 fn spawn_loot_roll_relay(coordinator: Coordinator) -> std::thread::JoinHandle<()> {
     std::thread::Builder::new()
@@ -2523,7 +2523,7 @@ impl Coordinator {
         coordinator.arm_shared_world_view()?;
         // The loot-roll relay: promotes each world shard's staging loot rolls onto realm-core and settles
         // resolved winners back down. A no-op loop on an unsharded gateway (`relay_tick` returns
-        // immediately when `realm_store()` is `None`), so this costs a single-database deployment
+        // immediately when `realm_store()` is `Ok(None)`), so this costs a single-database deployment
         // nothing but one idle thread.
         spawn_loot_roll_relay(coordinator.clone());
         // Member Stats for every group mate outside a viewer's AOI, on its own 5 s thread.
@@ -2683,8 +2683,8 @@ impl Coordinator {
             .collect()
     }
 
-    /// Check current subscription health before returning the authentication authority.
-    /// The configured connection map survives disconnection and may retain stale credentials.
+    /// Check current subscription health before returning Realm-core authority.
+    /// The configured connection map survives disconnection and may retain stale rows.
     pub(crate) fn realm_core(&self) -> Result<Coordinator> {
         let db = self
             .1
@@ -2697,15 +2697,12 @@ impl Coordinator {
             })
             .map_err(|db| {
                 anyhow!(
-                    "realm-core database {db} is not connected — refusing to authenticate against \
-                     the world database's stale auth cache"
+                    "Realm-core database {db} is unavailable; refusing to use World Shard authority"
                 )
             })?;
-        let inner = self
-            .1
-            .conns
-            .get(db)
-            .ok_or_else(|| anyhow!("auth database {db} missing from the coordinator set"))?;
+        let inner = self.1.conns.get(db).ok_or_else(|| {
+            anyhow!("Realm-core authority database {db} missing from the coordinator set")
+        })?;
         Ok(Coordinator(inner.clone(), self.1.clone(), self.2.clone()))
     }
 

gateway/src/stdb/store/loot_roll.rs

@@ -78,7 +78,7 @@ impl LootRollStore for Coordinator {
     /// coordinator subscription list, so no reducer call is
     /// needed. Meaningful only on a WORLD SHARD in a sharded deployment: realm-core never has a row
     /// here that a world shard wrote (only its own `realm_loot_op` START arm inserts there), and on
-    /// an unsharded gateway the relay that calls this never runs at all (`realm_store()` is `None`).
+    /// an unsharded gateway the relay that calls this never runs at all (`realm_store()` is `Ok(None)`).
     fn pending_local_rolls(&self) -> Result<Vec<PendingLootRoll>> {
         let guard = self.0.coord();
         let db = &guard.conn.db;

gateway/src/stdb/store/shard_routing.rs

@@ -101,46 +101,9 @@ impl ShardRoutingStore for Coordinator {
         self.establish_session(account_id, session_key, identity)
     }
 
-    /// The realm-core handle, but ONLY on a multi-database gateway.
-    ///
-    /// The `is_sharded()` short-circuit is the whole of "unset env vars ⇒ today's gateway" for the
-    /// realm-wide party plane: with one database `realm_core()` would answer THIS handle, and routing
-    /// party ops through the operator-gated realm reducers against the same database would be a
-    /// different code path, a different gate, and a different set of writes for no gain. Answering
-    /// `None` here sends `world::party` down the arm that calls exactly the reducers it called
-    /// before realm-core existed.
-    ///
-    /// Configured-but-unreachable realm-core answers `None` as well (`realm_core()` is `Err`), so a
-    /// realm-core outage degrades party ops to shard-local rather than failing them. That is a
-    /// deliberate split from the AUTH paths, which fail CLOSED: a stale auth cache lets someone in
-    /// with a revoked credential, while a shard-local party op is only ever a smaller party.
-    fn realm_store(&self) -> Option<std::sync::Arc<dyn WorldStore>> {
-        if !self.is_sharded() {
-            return None;
-        }
-        self.realm_core()
-            .ok()
-            .map(|rc| std::sync::Arc::new(rc) as std::sync::Arc<dyn WorldStore>)
-    }
-
-    fn party_cleanup_realm(&self) -> Result<Option<std::sync::Arc<dyn WorldStore>>> {
-        if !self.is_sharded() {
-            return Ok(None);
-        }
-        self.realm_core()
-            .map(|realm| Some(std::sync::Arc::new(realm) as std::sync::Arc<dyn WorldStore>))
-    }
-
-    fn party_command_realm(&self) -> Result<Option<std::sync::Arc<dyn WorldStore>>> {
-        if !self.is_sharded() {
-            return Ok(None);
-        }
-        self.realm_core()
-            .map(|realm| Some(std::sync::Arc::new(realm) as std::sync::Arc<dyn WorldStore>))
-    }
-
-    fn transfer_realm(&self) -> Result<Option<std::sync::Arc<dyn WorldStore>>> {
-        if !self.is_sharded() {
+    fn realm_store(&self) -> Result<Option<std::sync::Arc<dyn WorldStore>>> {
+        // A configured Realm-core may have failed its initial connection.
+        if !self.is_sharded() && self.shard_map().realm_core_db().is_none() {
             return Ok(None);
         }
         self.realm_core()

gateway/src/world/handle_loot_fake.rs

@@ -377,19 +377,7 @@ impl ShardRoutingStore for HandleLootFake {
         Ok(())
     }
 
-    fn realm_store(&self) -> Option<Arc<dyn WorldStore>> {
-        None
-    }
-
-    fn party_cleanup_realm(&self) -> Result<Option<Arc<dyn WorldStore>>> {
-        Ok(None)
-    }
-
-    fn party_command_realm(&self) -> Result<Option<Arc<dyn WorldStore>>> {
-        Ok(None)
-    }
-
-    fn transfer_realm(&self) -> Result<Option<Arc<dyn WorldStore>>> {
+    fn realm_store(&self) -> Result<Option<Arc<dyn WorldStore>>> {
         Ok(None)
     }
 

gateway/src/world/handlers/mail.rs

@@ -34,11 +34,13 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                             Box::new(codec::build_mail_list(&mails, mail::now_secs())),
                         )))
                     }
-                    Err(failure) => log::debug!(
-                        "world: mail list refused (account {}): {}",
-                        conn.account_id,
-                        failure.refusal()?
-                    ),
+                    Err(failure) => {
+                        let refusal = failure.refusal()?;
+                        log::debug!(
+                            "world: mail list refused (account {}): {refusal}",
+                            conn.account_id
+                        );
+                    }
                 }
             }
             // The periodic "do I have new mail" poll behind the minimap envelope. It names no mailbox —
@@ -49,10 +51,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                 let unread = match mail::has_unread(store, conn.actor()) {
                     Ok(unread) => unread,
                     Err(failure) => {
+                        let refusal = failure.refusal()?;
                         log::debug!(
-                            "world: mail poll answered empty (account {}): {}",
-                            conn.account_id,
-                            failure.refusal()?
+                            "world: mail poll answered empty (account {}): {refusal}",
+                            conn.account_id
                         );
                         false
                     }
@@ -81,10 +83,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                 ) {
                     Ok(body) => body,
                     Err(failure) => {
+                        let refusal = failure.refusal()?;
                         log::debug!(
-                            "world: item text query refused (account {}): {}",
-                            conn.account_id,
-                            failure.refusal()?
+                            "world: item text query refused (account {}): {refusal}",
+                            conn.account_id
                         );
                         None
                     }
@@ -104,10 +106,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                 if let Err(failure) =
                     mail::mark_read(store, actor, c.mailbox.guid(), u64::from(c.mail_id))
                 {
+                    let refusal = failure.refusal()?;
                     log::debug!(
-                        "world: mail mark-as-read refused (account {}): {}",
-                        conn.account_id,
-                        failure.refusal()?
+                        "world: mail mark-as-read refused (account {}): {refusal}",
+                        conn.account_id
                     );
                 }
             }
@@ -121,10 +123,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                 {
                     Ok(()) => true,
                     Err(failure) => {
+                        let refusal = failure.refusal()?;
                         log::debug!(
-                            "world: mail delete refused (account {}): {}",
-                            conn.account_id,
-                            failure.refusal()?
+                            "world: mail delete refused (account {}): {refusal}",
+                            conn.account_id
                         );
                         false
                     }
@@ -147,10 +149,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                 ) {
                     Ok(()) => true,
                     Err(failure) => {
+                        let refusal = failure.refusal()?;
                         log::debug!(
-                            "world: mail return refused (account {}): {}",
-                            conn.account_id,
-                            failure.refusal()?
+                            "world: mail return refused (account {}): {refusal}",
+                            conn.account_id
                         );
                         false
                     }
@@ -170,10 +172,10 @@ impl<St: CharacterStore + MailStore + SessionStore + ShardRoutingStore + SocialS
                     match mail::take_money(store, actor, c.mailbox.guid(), u64::from(c.mail_id)) {
                         Ok(()) => true,
                         Err(failure) => {
+                            let refusal = failure.refusal()?;
                             log::debug!(
-                                "world: mail take-money refused (account {}): {}",
-                                conn.account_id,
-                                failure.refusal()?
+                                "world: mail take-money refused (account {}): {refusal}",
+                                conn.account_id
                             );
                             false
                         }

gateway/src/world/loot.rs

@@ -108,7 +108,7 @@ pub(crate) fn run_vote<St: LootRollStore + ShardRoutingStore + ?Sized>(
     slot: u32,
     vote: u8,
 ) -> Result<LootActionStatus> {
-    let Some(realm) = store.realm_store() else {
+    let Some(realm) = store.realm_store()? else {
         return store.loot_roll(actor, corpse_guid, slot, vote);
     };
     flush_pending_promotions(store, realm.as_ref());
@@ -190,7 +190,7 @@ pub(crate) fn flush_pending_promotions<St: ShardRoutingStore + ?Sized>(
 }
 
 /// One promotion + settlement pass. No-op on an unsharded store (`realm_store()` answers
-/// `None`), which is what makes running this on a timer free for a single-database gateway.
+/// `Ok(None)`), which is what makes running this on a timer free for a single-database gateway.
 ///
 /// `won_watermark` advances only after every settlement in the batch succeeds on every Shard.
 /// A failure leaves the result available to retry while its Realm-core event still exists.
@@ -206,8 +206,13 @@ pub(crate) fn flush_pending_promotions<St: ShardRoutingStore + ?Sized>(
 /// function's own caller's poll interval, not by anything in here — see this module's doc for why
 /// that caller is a poll rather than a persistent callback.
 pub(crate) fn relay_tick<St: ShardRoutingStore + ?Sized>(store: &St, won_watermark: &mut u64) {
-    let Some(realm) = store.realm_store() else {
-        return;
+    let realm = match store.realm_store() {
+        Ok(Some(realm)) => realm,
+        Ok(None) => return,
+        Err(error) => {
+            log::warn!("loot-roll Relay waits for Realm-core: {error:#}");
+            return;
+        }
     };
     flush_pending_promotions(store, realm.as_ref());
 

gateway/src/world/loot_tests.rs

@@ -15,6 +15,64 @@ use lyracore_shared::loot_roll::loot_op;
 
 const SOURCE: spacetimedb_sdk::Identity = spacetimedb_sdk::Identity::from_byte_array([7; 32]);
 
+#[test]
+fn a_loot_vote_during_realm_core_loss_cannot_vote_locally() {
+    let (realm, world, _instances, _calls) = party_topology();
+    *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into());
+
+    let error = loot::run_vote(
+        world.as_ref(),
+        Actor::new(GINGER).unwrap(),
+        500,
+        2,
+        lyracore_shared::loot_roll::vote_kind::NEED,
+    )
+    .expect_err("Realm-core still owns the Loot Roll");
+
+    assert_eq!(
+        crate::stdb::classify(&error),
+        crate::stdb::DurableFailure::TransportLoss
+    );
+    assert!(world.loot_roll.loot_rolls.lock().unwrap().is_empty());
+    assert!(realm.loot_roll.realm_loot_ops.lock().unwrap().is_empty());
+}
+
+#[test]
+fn loot_promotion_waits_for_realm_core_recovery() {
+    let (realm, world, _instances, _calls) = party_topology();
+    world
+        .loot_roll
+        .pending_rolls
+        .lock()
+        .unwrap()
+        .push(loot::PendingLootRoll {
+            promotion_source: SOURCE,
+            roll_id: 77,
+            corpse_guid: 500,
+            slot: 2,
+            item_entry: 1234,
+            deadline_micros: 999_999,
+            recipients: vec![GINGER, TRIN],
+            random_property_id: 0,
+        });
+    *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into());
+    let mut watermark = 0;
+
+    loot::relay_tick(world.as_ref(), &mut watermark);
+
+    assert_eq!(watermark, 0);
+    assert_eq!(world.loot_roll.pending_rolls.lock().unwrap().len(), 1);
+    assert!(world.loot_roll.cleared_rolls.lock().unwrap().is_empty());
+    assert!(realm.loot_roll.realm_loot_ops.lock().unwrap().is_empty());
+
+    *world.topology.realm_error.lock().unwrap() = None;
+    loot::relay_tick(world.as_ref(), &mut watermark);
+    loot::relay_tick(world.as_ref(), &mut watermark);
+
+    assert!(world.loot_roll.pending_rolls.lock().unwrap().is_empty());
+    assert_eq!(realm.loot_roll.realm_loot_ops.lock().unwrap().len(), 1);
+}
+
 // ---- `run_vote` (CMSG_LOOT_ROLL routing) ----
 
 /// Unsharded → the shard-local loot path, verbatim: the vote runs on the player's own reducer, and
@@ -118,7 +176,7 @@ fn a_vote_on_a_staged_roll_reaches_realm_core_after_its_promotion() {
 
 // ---- `relay_tick` (promotion + settlement) ----
 
-/// Unsharded → no-op, before even reading a shard's pending rolls: `realm_store()` answers `None`
+/// Unsharded → no-op, before even reading a shard's pending rolls: `realm_store()` answers `Ok(None)`
 /// and `relay_tick` returns before `world_stores()` is ever consulted.
 #[test]
 fn an_unsharded_store_does_nothing_even_with_pending_rolls_on_a_connected_peer() {