From a32f6928bc97344f0d8326395bc5b028cdad9407 Mon Sep 17 00:00:00 2001 From: Mathias Hartveit Date: Sun, 11 Oct 2026 22:25:51 +0200 Subject: [PATCH] fix(gateway): keep Realm-core authority during outages --- CONTEXT.md | 2 + docs/architecture.md | 10 ++ gateway/src/stdb/connection.rs | 19 ++- gateway/src/stdb/store/loot_roll.rs | 2 +- gateway/src/stdb/store/shard_routing.rs | 43 +----- gateway/src/world/handle_loot_fake.rs | 14 +- gateway/src/world/handlers/mail.rs | 48 +++---- gateway/src/world/loot.rs | 13 +- gateway/src/world/loot_tests.rs | 60 ++++++++- gateway/src/world/mail.rs | 40 ++++-- gateway/src/world/mail_tests.rs | 144 ++++++++++++++++++++- gateway/src/world/party.rs | 30 +++-- gateway/src/world/party_mirror.rs | 2 +- gateway/src/world/party_tests.rs | 34 ++++- gateway/src/world/presence.rs | 12 +- gateway/src/world/presence_tests.rs | 30 +++++ gateway/src/world/store.rs | 21 +-- gateway/src/world/test_support/topology.rs | 43 ++---- gateway/src/world/transfer.rs | 2 +- gateway/src/world/transfer_tests.rs | 3 +- 20 files changed, 392 insertions(+), 180 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index a638dab2..4214a755 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -530,6 +530,8 @@ Status), in transit between two places (a pending Transfer, or a Shard's own row with no entity there), or offline. A negative Whereabouts, offline or no Character found at all, needs every configured World Shard to vouch that none of them is hiding the Character. When one cannot, the Whereabouts is unknown: another Shard's health, not a Transport Loss. +An unavailable Realm-core also leaves negative Whereabouts unknown because the Gateway cannot +exclude a pending Transfer. _Avoid_: presence state, location status **Away Status**: diff --git a/docs/architecture.md b/docs/architecture.md index d7b1ae0f..c23233bb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,6 +98,11 @@ routing fact; module game logic never reads one, and an architecture test fails `gateway/src/stdb/store/.rs` calls the reducer directly. When the call fails, `classify` in `gateway/src/stdb/connection.rs` decides. A Refusal gets the family's answer and the session continues. Every other failure is a Transport Loss and ends the World Session. +- **Realm-core routing preserves authority.** `ShardRoutingStore::realm_store` returns the local + case only for a single-Shard Realm. An unavailable configured Realm-core returns an infrastructure + failure, so Mail, party and Loot Roll operations cannot fall back to a World Shard. Cleanup, + Companion Orders and Transfer use the same routing contract. A Realm Presence read that cannot + exclude a pending Transfer reports unknown Whereabouts. ### 2.3 The honest exception: gates that live in the gateway @@ -622,6 +627,11 @@ Afterwards the gateway re-pins the session: new home handle, `bind_shard_session ### 6.3b Mail attachments across the boundary +Mail recovery at world entry and after quest completion is best effort. If Realm-core is unavailable, +recovery logs the failure and leaves Escrow pending for the next recovery attempt. Explicit Mail +requests still end the World Session on Transport Loss. Recovery never commits Mail to another +Shard because its authority is unavailable (`gateway/src/world/mail.rs`). + The same problem with different nouns: a mail row is authoritative on realm-core, the sender's purse and items are on their own shard, and no transaction spans the two. `module/src/mail_escrow.rs` answers it with the same escrow shape — `realm_mail_fence` (the value leaves the purse INTO the diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index 7b946bb3..56e8799f 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -1248,7 +1248,7 @@ const LOOT_ROLL_RELAY_POLL: Duration = Duration::from_millis(200); /// Background loop driving `world::loot::relay_tick`: promotes each world shard's freshly /// staged loot rolls onto realm-core, and settles resolved winners back down. A no-op on an unsharded -/// gateway — `relay_tick` returns immediately when `Coordinator::realm_store()` answers `None` — so +/// gateway — `relay_tick` returns immediately when `Coordinator::realm_store()` answers `Ok(None)` — so /// this thread costs a single-database deployment nothing but its own idle wakeups. fn spawn_loot_roll_relay(coordinator: Coordinator) -> std::thread::JoinHandle<()> { std::thread::Builder::new() @@ -2523,7 +2523,7 @@ impl Coordinator { coordinator.arm_shared_world_view()?; // The loot-roll relay: promotes each world shard's staging loot rolls onto realm-core and settles // resolved winners back down. A no-op loop on an unsharded gateway (`relay_tick` returns - // immediately when `realm_store()` is `None`), so this costs a single-database deployment + // immediately when `realm_store()` is `Ok(None)`), so this costs a single-database deployment // nothing but one idle thread. spawn_loot_roll_relay(coordinator.clone()); // Member Stats for every group mate outside a viewer's AOI, on its own 5 s thread. @@ -2683,8 +2683,8 @@ impl Coordinator { .collect() } - /// Check current subscription health before returning the authentication authority. - /// The configured connection map survives disconnection and may retain stale credentials. + /// Check current subscription health before returning Realm-core authority. + /// The configured connection map survives disconnection and may retain stale rows. pub(crate) fn realm_core(&self) -> Result { let db = self .1 @@ -2697,15 +2697,12 @@ impl Coordinator { }) .map_err(|db| { anyhow!( - "realm-core database {db} is not connected — refusing to authenticate against \ - the world database's stale auth cache" + "Realm-core database {db} is unavailable; refusing to use World Shard authority" ) })?; - let inner = self - .1 - .conns - .get(db) - .ok_or_else(|| anyhow!("auth database {db} missing from the coordinator set"))?; + let inner = self.1.conns.get(db).ok_or_else(|| { + anyhow!("Realm-core authority database {db} missing from the coordinator set") + })?; Ok(Coordinator(inner.clone(), self.1.clone(), self.2.clone())) } diff --git a/gateway/src/stdb/store/loot_roll.rs b/gateway/src/stdb/store/loot_roll.rs index 7bb17348..65c1089d 100644 --- a/gateway/src/stdb/store/loot_roll.rs +++ b/gateway/src/stdb/store/loot_roll.rs @@ -78,7 +78,7 @@ impl LootRollStore for Coordinator { /// coordinator subscription list, so no reducer call is /// needed. Meaningful only on a WORLD SHARD in a sharded deployment: realm-core never has a row /// here that a world shard wrote (only its own `realm_loot_op` START arm inserts there), and on - /// an unsharded gateway the relay that calls this never runs at all (`realm_store()` is `None`). + /// an unsharded gateway the relay that calls this never runs at all (`realm_store()` is `Ok(None)`). fn pending_local_rolls(&self) -> Result> { let guard = self.0.coord(); let db = &guard.conn.db; diff --git a/gateway/src/stdb/store/shard_routing.rs b/gateway/src/stdb/store/shard_routing.rs index e95dc30c..cd9a38e1 100644 --- a/gateway/src/stdb/store/shard_routing.rs +++ b/gateway/src/stdb/store/shard_routing.rs @@ -101,46 +101,9 @@ impl ShardRoutingStore for Coordinator { self.establish_session(account_id, session_key, identity) } - /// The realm-core handle, but ONLY on a multi-database gateway. - /// - /// The `is_sharded()` short-circuit is the whole of "unset env vars ⇒ today's gateway" for the - /// realm-wide party plane: with one database `realm_core()` would answer THIS handle, and routing - /// party ops through the operator-gated realm reducers against the same database would be a - /// different code path, a different gate, and a different set of writes for no gain. Answering - /// `None` here sends `world::party` down the arm that calls exactly the reducers it called - /// before realm-core existed. - /// - /// Configured-but-unreachable realm-core answers `None` as well (`realm_core()` is `Err`), so a - /// realm-core outage degrades party ops to shard-local rather than failing them. That is a - /// deliberate split from the AUTH paths, which fail CLOSED: a stale auth cache lets someone in - /// with a revoked credential, while a shard-local party op is only ever a smaller party. - fn realm_store(&self) -> Option> { - if !self.is_sharded() { - return None; - } - self.realm_core() - .ok() - .map(|rc| std::sync::Arc::new(rc) as std::sync::Arc) - } - - fn party_cleanup_realm(&self) -> Result>> { - if !self.is_sharded() { - return Ok(None); - } - self.realm_core() - .map(|realm| Some(std::sync::Arc::new(realm) as std::sync::Arc)) - } - - fn party_command_realm(&self) -> Result>> { - if !self.is_sharded() { - return Ok(None); - } - self.realm_core() - .map(|realm| Some(std::sync::Arc::new(realm) as std::sync::Arc)) - } - - fn transfer_realm(&self) -> Result>> { - if !self.is_sharded() { + fn realm_store(&self) -> Result>> { + // A configured Realm-core may have failed its initial connection. + if !self.is_sharded() && self.shard_map().realm_core_db().is_none() { return Ok(None); } self.realm_core() diff --git a/gateway/src/world/handle_loot_fake.rs b/gateway/src/world/handle_loot_fake.rs index c742d6d3..b7ec9522 100644 --- a/gateway/src/world/handle_loot_fake.rs +++ b/gateway/src/world/handle_loot_fake.rs @@ -377,19 +377,7 @@ impl ShardRoutingStore for HandleLootFake { Ok(()) } - fn realm_store(&self) -> Option> { - None - } - - fn party_cleanup_realm(&self) -> Result>> { - Ok(None) - } - - fn party_command_realm(&self) -> Result>> { - Ok(None) - } - - fn transfer_realm(&self) -> Result>> { + fn realm_store(&self) -> Result>> { Ok(None) } diff --git a/gateway/src/world/handlers/mail.rs b/gateway/src/world/handlers/mail.rs index b9b1b573..eff9ca95 100644 --- a/gateway/src/world/handlers/mail.rs +++ b/gateway/src/world/handlers/mail.rs @@ -34,11 +34,13 @@ impl log::debug!( - "world: mail list refused (account {}): {}", - conn.account_id, - failure.refusal()? - ), + Err(failure) => { + let refusal = failure.refusal()?; + log::debug!( + "world: mail list refused (account {}): {refusal}", + conn.account_id + ); + } } } // The periodic "do I have new mail" poll behind the minimap envelope. It names no mailbox — @@ -49,10 +51,10 @@ impl unread, Err(failure) => { + let refusal = failure.refusal()?; log::debug!( - "world: mail poll answered empty (account {}): {}", - conn.account_id, - failure.refusal()? + "world: mail poll answered empty (account {}): {refusal}", + conn.account_id ); false } @@ -81,10 +83,10 @@ impl body, Err(failure) => { + let refusal = failure.refusal()?; log::debug!( - "world: item text query refused (account {}): {}", - conn.account_id, - failure.refusal()? + "world: item text query refused (account {}): {refusal}", + conn.account_id ); None } @@ -104,10 +106,10 @@ impl true, Err(failure) => { + let refusal = failure.refusal()?; log::debug!( - "world: mail delete refused (account {}): {}", - conn.account_id, - failure.refusal()? + "world: mail delete refused (account {}): {refusal}", + conn.account_id ); false } @@ -147,10 +149,10 @@ impl true, Err(failure) => { + let refusal = failure.refusal()?; log::debug!( - "world: mail return refused (account {}): {}", - conn.account_id, - failure.refusal()? + "world: mail return refused (account {}): {refusal}", + conn.account_id ); false } @@ -170,10 +172,10 @@ impl true, Err(failure) => { + let refusal = failure.refusal()?; log::debug!( - "world: mail take-money refused (account {}): {}", - conn.account_id, - failure.refusal()? + "world: mail take-money refused (account {}): {refusal}", + conn.account_id ); false } diff --git a/gateway/src/world/loot.rs b/gateway/src/world/loot.rs index 2aecbf07..d0436f8b 100644 --- a/gateway/src/world/loot.rs +++ b/gateway/src/world/loot.rs @@ -108,7 +108,7 @@ pub(crate) fn run_vote( slot: u32, vote: u8, ) -> Result { - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { return store.loot_roll(actor, corpse_guid, slot, vote); }; flush_pending_promotions(store, realm.as_ref()); @@ -190,7 +190,7 @@ pub(crate) fn flush_pending_promotions( } /// One promotion + settlement pass. No-op on an unsharded store (`realm_store()` answers -/// `None`), which is what makes running this on a timer free for a single-database gateway. +/// `Ok(None)`), which is what makes running this on a timer free for a single-database gateway. /// /// `won_watermark` advances only after every settlement in the batch succeeds on every Shard. /// A failure leaves the result available to retry while its Realm-core event still exists. @@ -206,8 +206,13 @@ pub(crate) fn flush_pending_promotions( /// function's own caller's poll interval, not by anything in here — see this module's doc for why /// that caller is a poll rather than a persistent callback. pub(crate) fn relay_tick(store: &St, won_watermark: &mut u64) { - let Some(realm) = store.realm_store() else { - return; + let realm = match store.realm_store() { + Ok(Some(realm)) => realm, + Ok(None) => return, + Err(error) => { + log::warn!("loot-roll Relay waits for Realm-core: {error:#}"); + return; + } }; flush_pending_promotions(store, realm.as_ref()); diff --git a/gateway/src/world/loot_tests.rs b/gateway/src/world/loot_tests.rs index ac1f8e1b..aeb14f59 100644 --- a/gateway/src/world/loot_tests.rs +++ b/gateway/src/world/loot_tests.rs @@ -15,6 +15,64 @@ use lyracore_shared::loot_roll::loot_op; const SOURCE: spacetimedb_sdk::Identity = spacetimedb_sdk::Identity::from_byte_array([7; 32]); +#[test] +fn a_loot_vote_during_realm_core_loss_cannot_vote_locally() { + let (realm, world, _instances, _calls) = party_topology(); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + let error = loot::run_vote( + world.as_ref(), + Actor::new(GINGER).unwrap(), + 500, + 2, + lyracore_shared::loot_roll::vote_kind::NEED, + ) + .expect_err("Realm-core still owns the Loot Roll"); + + assert_eq!( + crate::stdb::classify(&error), + crate::stdb::DurableFailure::TransportLoss + ); + assert!(world.loot_roll.loot_rolls.lock().unwrap().is_empty()); + assert!(realm.loot_roll.realm_loot_ops.lock().unwrap().is_empty()); +} + +#[test] +fn loot_promotion_waits_for_realm_core_recovery() { + let (realm, world, _instances, _calls) = party_topology(); + world + .loot_roll + .pending_rolls + .lock() + .unwrap() + .push(loot::PendingLootRoll { + promotion_source: SOURCE, + roll_id: 77, + corpse_guid: 500, + slot: 2, + item_entry: 1234, + deadline_micros: 999_999, + recipients: vec![GINGER, TRIN], + random_property_id: 0, + }); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + let mut watermark = 0; + + loot::relay_tick(world.as_ref(), &mut watermark); + + assert_eq!(watermark, 0); + assert_eq!(world.loot_roll.pending_rolls.lock().unwrap().len(), 1); + assert!(world.loot_roll.cleared_rolls.lock().unwrap().is_empty()); + assert!(realm.loot_roll.realm_loot_ops.lock().unwrap().is_empty()); + + *world.topology.realm_error.lock().unwrap() = None; + loot::relay_tick(world.as_ref(), &mut watermark); + loot::relay_tick(world.as_ref(), &mut watermark); + + assert!(world.loot_roll.pending_rolls.lock().unwrap().is_empty()); + assert_eq!(realm.loot_roll.realm_loot_ops.lock().unwrap().len(), 1); +} + // ---- `run_vote` (CMSG_LOOT_ROLL routing) ---- /// Unsharded → the shard-local loot path, verbatim: the vote runs on the player's own reducer, and @@ -118,7 +176,7 @@ fn a_vote_on_a_staged_roll_reaches_realm_core_after_its_promotion() { // ---- `relay_tick` (promotion + settlement) ---- -/// Unsharded → no-op, before even reading a shard's pending rolls: `realm_store()` answers `None` +/// Unsharded → no-op, before even reading a shard's pending rolls: `realm_store()` answers `Ok(None)` /// and `relay_tick` returns before `world_stores()` is ever consulted. #[test] fn an_unsharded_store_does_nothing_even_with_pending_rolls_on_a_connected_peer() { diff --git a/gateway/src/world/mail.rs b/gateway/src/world/mail.rs index 74a028fb..bae9f6d6 100644 --- a/gateway/src/world/mail.rs +++ b/gateway/src/world/mail.rs @@ -388,7 +388,7 @@ pub(crate) fn mail_of( store: &St, self_guid: u64, ) -> Result> { - let mails = match store.realm_store() { + let mails = match store.realm_store()? { Some(realm) => realm.mail_list(self_guid)?, None => store.mail_list(self_guid)?, }; @@ -436,7 +436,7 @@ pub(crate) fn mark_read( mail_id: u64, ) -> MailResult<()> { let actor = at_mailbox(store, actor, mailbox_guid)?; - match store.realm_store() { + match store.realm_store()? { Some(realm) => realm.mail_mark_read(actor, mail_id), None => store.mail_mark_read(actor, mail_id), }?; @@ -449,7 +449,7 @@ pub(crate) fn delete( mail_id: u64, ) -> MailResult<()> { let actor = at_mailbox(store, actor, mailbox_guid)?; - match store.realm_store() { + match store.realm_store()? { Some(realm) => realm.mail_delete(actor, mail_id), None => store.mail_delete(actor, mail_id), }?; @@ -462,7 +462,7 @@ pub(crate) fn return_to_sender( mail_id: u64, ) -> MailResult<()> { let actor = at_mailbox(store, actor, mailbox_guid)?; - let realm = store.realm_store(); + let realm = store.realm_store()?; let mail = match &realm { Some(realm) => realm.mail_by_id(mail_id)?, None => store.mail_by_id(mail_id)?, @@ -568,7 +568,7 @@ pub(crate) fn send< }; let cod = mail_rules::cod_at_send(cod, item_guid != 0); let same_account = same_realm_account(store, sender_guid, recipient_guid)?; - match store.realm_store() { + match store.realm_store()? { None => store.mail_send( sender_actor, recipient_guid, @@ -704,8 +704,18 @@ pub(crate) fn redrive(store: &St, se let Some(actor) = Actor::new(self_guid) else { return; }; - let realm = store.realm_store(); - for held in store.mail_escrows_of(self_guid).unwrap_or_default() { + let realm = match store.realm_store() { + Ok(realm) => realm, + Err(error) => { + log::warn!("mail: recovery for Character {self_guid} waits for Realm-core: {error:#}"); + return; + } + }; + let outgoing = store.mail_escrows_of(self_guid).unwrap_or_else(|error| { + log::warn!("mail: cannot read outgoing Escrow for Character {self_guid}: {error:#}"); + Vec::new() + }); + for held in outgoing { if held.payout { continue; // A payout fence never lives on a shard; ignore a stray rather than mis-drive it. } @@ -723,7 +733,11 @@ pub(crate) fn redrive(store: &St, se let Some(realm) = realm else { return; // One database: a take is one transaction and files no Escrow. }; - for held in realm.mail_escrows_of(self_guid).unwrap_or_default() { + let payouts = realm.mail_escrows_of(self_guid).unwrap_or_else(|error| { + log::warn!("mail: cannot read payout Escrow for Character {self_guid}: {error:#}"); + Vec::new() + }); + for held in payouts { if !held.payout { continue; } @@ -770,7 +784,7 @@ pub(crate) fn take_money( mail_id: u64, ) -> MailResult<()> { let actor = at_mailbox(store, actor, mailbox_guid)?; - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { return Ok(store.mail_take_money(actor, mail_id)?); }; let amount = mail_of(store, actor.guid())? @@ -832,7 +846,7 @@ pub(crate) fn take_item( return Err(refused(mail_rules::NOTHING_TO_TAKE)); } let taken = (row.item_entry, row.item_stack_count); - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { store.mail_take_item(actor, mail_id)?; return Ok(taken); }; @@ -982,7 +996,7 @@ pub(crate) fn copy_letter( let actor = at_mailbox::(store, actor, mailbox_guid) .map_err(|failure| failure.map(CopyLetterRefusal::NoMailbox))?; store.mail_item_room(actor)?; - match store.realm_store() { + match store.realm_store()? { Some(realm) => realm.mail_copy_text(actor, mail_id), None => store.mail_copy_text(actor, mail_id), }?; @@ -990,7 +1004,7 @@ pub(crate) fn copy_letter( // non-empty-body case) — the copy above just proved the body is non-empty. let item_text_id = u32::try_from(mail_id).unwrap_or(0); store.mail_grant_letter(actor, item_text_id)?; - match store.realm_store() { + match store.realm_store()? { Some(realm) => realm.mail_mark_letter_granted(actor, mail_id), None => store.mail_mark_letter_granted(actor, mail_id), }?; @@ -1029,7 +1043,7 @@ pub(crate) fn item_text( if !owns_item && own_mail_body.is_none() { return Ok(None); } - let copied = match store.realm_store() { + let copied = match store.realm_store()? { Some(realm) => realm.item_text(item_text_id), None => store.item_text(item_text_id), }?; diff --git a/gateway/src/world/mail_tests.rs b/gateway/src/world/mail_tests.rs index b3807662..e329eecb 100644 --- a/gateway/src/world/mail_tests.rs +++ b/gateway/src/world/mail_tests.rs @@ -172,7 +172,7 @@ fn the_realm_plane_and_the_single_database_fallback_render_the_same_mailbox() { fn an_unsharded_gateway_reads_the_mailbox_on_its_own_database() { let single = unsharded_mailbox(); assert!( - single.realm_store().is_none(), + single.realm_store().unwrap().is_none(), "fixture: this is the single-database gateway" ); @@ -4051,6 +4051,13 @@ fn a_reward_letter_whose_drive_died_after_the_commit_is_delivered_once() { ); *world.mail.mail_kill_at.lock().unwrap() = None; + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + mail::redrive(world.as_ref(), GINGER); + assert!(world.mail.mails.lock().unwrap().is_empty()); + assert_eq!(world.mail.mail_escrows.lock().unwrap().len(), 1); + assert_eq!(reward_letters(&realm, GINGER).len(), 1); + + *world.topology.realm_error.lock().unwrap() = None; let visible = mail::open_mailbox(world.as_ref(), Actor::new(GINGER), MAILBOX).expect("the gate opens"); @@ -4069,6 +4076,49 @@ fn a_reward_letter_whose_drive_died_after_the_commit_is_delivered_once() { ); } +#[test] +fn a_reward_letter_waits_for_realm_core_without_changing_its_escrow() { + let (realm, world, _instances, _calls) = sharded_send(); + let held = card_renewal_letter(GINGER, 0); + hold_reward_letter(&world, GINGER, held.clone()); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + mail::redrive(world.as_ref(), GINGER); + + assert_eq!( + *world.mail.mail_escrows.lock().unwrap(), + vec![(GINGER, held)] + ); + assert!(world.mail.mails.lock().unwrap().is_empty()); + assert!(realm.mail.mails.lock().unwrap().is_empty()); + + *world.topology.realm_error.lock().unwrap() = None; + mail::redrive(world.as_ref(), GINGER); + mail::redrive(world.as_ref(), GINGER); + + let letters = reward_letters(&realm, GINGER); + assert_eq!(letters.len(), 1); + assert_card_renewal(&letters[0]); + assert!(world.mail.mail_escrows.lock().unwrap().is_empty()); + assert!(world.mail.mails.lock().unwrap().is_empty()); +} + +#[test] +fn a_realm_core_outage_before_send_preserves_the_item_and_copper() { + let (realm, world, _instances, _calls) = sharded_send(); + give_item(&world, GINGER, SWORD_GUID, sword()); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + let failure = post_item(world.as_ref(), "Trin").expect_err("Mail authority is unavailable"); + + assert!(matches!(failure, mail::MailFailure::Lost(_))); + assert_eq!(world.bags_of(GINGER), vec![sword()]); + assert_eq!(*world.mail.purses.lock().unwrap(), vec![(GINGER, PURSE)]); + assert!(world.mail.mail_escrows.lock().unwrap().is_empty()); + assert!(world.mail.mails.lock().unwrap().is_empty()); + assert!(realm.mail.mails.lock().unwrap().is_empty()); +} + #[test] fn a_single_database_drives_a_reward_letter_on_its_own_database() { let store = unsharded_send(); @@ -4225,3 +4275,95 @@ fn a_send_refusal_is_answered_and_a_transport_loss_ends_the_session() { *realm.mail.mail_kill_at.lock().unwrap() = Some("mail_commit".into()); assert!(Mail::handle(world.as_ref(), &mut session, send("Trin").into()).is_err()); } + +#[test] +fn explicit_mail_requests_propagate_realm_core_loss() { + use wow_world_messages::vanilla::{ + CMSG_GET_MAIL_LIST, CMSG_ITEM_TEXT_QUERY, CMSG_MAIL_DELETE, CMSG_MAIL_MARK_AS_READ, + CMSG_MAIL_RETURN_TO_SENDER, CMSG_MAIL_TAKE_MONEY, + }; + + let (_realm, world, _calls) = sharded_mailbox(); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + let mailbox = Guid::new(MAILBOX); + let requests = [ + ClientOpcodeMessage::CMSG_GET_MAIL_LIST(CMSG_GET_MAIL_LIST { mailbox }), + ClientOpcodeMessage::MSG_QUERY_NEXT_MAIL_TIME, + ClientOpcodeMessage::CMSG_ITEM_TEXT_QUERY(Box::new(CMSG_ITEM_TEXT_QUERY { + item_text_id: 1, + mail_id: 1, + unknown1: 0, + })), + ClientOpcodeMessage::CMSG_MAIL_MARK_AS_READ(Box::new(CMSG_MAIL_MARK_AS_READ { + mailbox, + mail_id: 1, + })), + ClientOpcodeMessage::CMSG_MAIL_DELETE(Box::new(CMSG_MAIL_DELETE { + mailbox_id: mailbox, + mail_id: 1, + })), + ClientOpcodeMessage::CMSG_MAIL_RETURN_TO_SENDER(Box::new(CMSG_MAIL_RETURN_TO_SENDER { + mailbox_id: mailbox, + mail_id: 1, + })), + ClientOpcodeMessage::CMSG_MAIL_TAKE_MONEY(Box::new(CMSG_MAIL_TAKE_MONEY { + mailbox, + mail_id: 1, + })), + ]; + + for request in requests { + let name = request.to_string(); + let mut session = ProtocolSession::in_world(7, GINGER); + let error = Mail::handle(world.as_ref(), &mut session, request.into()) + .err() + .unwrap_or_else(|| panic!("{name} must propagate Transport Loss")); + assert_eq!( + crate::stdb::classify(&error), + crate::stdb::DurableFailure::TransportLoss, + "{name}" + ); + } +} + +#[test] +fn a_mail_poll_after_realm_core_loss_ends_the_admitted_world_session() { + let mut home = std::sync::Arc::try_unwrap(seated_store()).ok().unwrap(); + home.topology.realm = Some(std::sync::Arc::new(WorldFake::default())); + let store = std::sync::Arc::new(home); + let queue = std::sync::Arc::new(LoginQueue::new(1, 0)); + let (mut client, socket) = world_session_socket_pair(); + let (finished, result) = std::sync::mpsc::channel(); + let server = { + let store = store.clone(); + let queue = queue.clone(); + std::thread::spawn(move || { + finished + .send(run_world_session_with_queue(socket, store, &queue)) + .unwrap(); + }) + }; + let (mut encrypt, mut decrypt) = client_handshake(&mut client, "TESTER", K); + CMSG_PLAYER_LOGIN { guid: Guid::new(1) } + .write_encrypted_client(&mut client, &mut encrypt) + .unwrap(); + drain_world_entry(&mut client, &mut decrypt); + assert_eq!(queue.active(), 1); + + *store.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + ClientOpcodeMessage::MSG_QUERY_NEXT_MAIL_TIME + .write_encrypted_client(&mut client, &mut encrypt) + .unwrap(); + + let error = result + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("the Mail poll must end the World Session") + .expect_err("an unavailable Mail authority is a Transport Loss"); + assert_eq!( + crate::stdb::classify(&error), + crate::stdb::DurableFailure::TransportLoss + ); + assert_eq!(queue.active(), 0); + assert!(ServerOpcodeMessage::read_encrypted(&mut client, &mut decrypt).is_err()); + server.join().unwrap(); +} diff --git a/gateway/src/world/party.rs b/gateway/src/world/party.rs index 62ff4a81..41cda5ce 100644 --- a/gateway/src/world/party.rs +++ b/gateway/src/world/party.rs @@ -869,7 +869,7 @@ pub(crate) fn run_party_command_intent( } let owned_realm; - let realm: &dyn WorldStore = match source.party_command_realm()? { + let realm: &dyn WorldStore = match source.realm_store()? { Some(handle) => { owned_realm = handle; owned_realm.as_ref() @@ -982,7 +982,7 @@ fn own_group_roster( store: &St, self_guid: u64, ) -> Result> { - match store.realm_store() { + match store.realm_store()? { Some(realm) => realm.group_roster(self_guid), None => store.group_roster(self_guid), } @@ -1135,7 +1135,7 @@ pub(crate) fn run< return Ok(GroupRefusal::TargetOffline.into()); } } - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { return match op { Op::Invite(target) => store.group_invite(actor, target), Op::Accept => store.group_accept(actor), @@ -1364,7 +1364,7 @@ pub(crate) fn run_bot_invite( return Ok(GroupRefusal::ActorUnavailable.into()); }; let owned_realm; - let realm: &dyn WorldStore = match store.realm_store() { + let realm: &dyn WorldStore = match store.realm_store()? { Some(r) => { owned_realm = r; owned_realm.as_ref() @@ -1404,7 +1404,7 @@ pub(crate) fn run_bot_leave(store: &St, leaver_guid: u64) -> Res return Ok(GroupRefusal::ActorUnavailable.into()); }; let owned_realm; - let realm: &dyn WorldStore = match store.realm_store() { + let realm: &dyn WorldStore = match store.realm_store()? { Some(r) => { owned_realm = r; owned_realm.as_ref() @@ -1502,7 +1502,7 @@ pub(crate) fn cleanup_deleted_character( store: &St, character_guid: u64, ) -> Result { - let Some(realm) = store.party_cleanup_realm()? else { + let Some(realm) = store.realm_store()? else { return Ok(DeletedCharacterPartyCleanup::AlreadyClean); }; // Guid 0 names no Character, so no party holds it. @@ -1546,7 +1546,7 @@ pub(crate) fn cleanup_deleted_character( pub(crate) fn reconcile_deleted_character_parties( store: &St, ) -> Result<()> { - let Some(realm) = store.party_cleanup_realm()? else { + let Some(realm) = store.realm_store()? else { return Ok(()); }; let mut failures = 0usize; @@ -1756,8 +1756,16 @@ pub(crate) fn on_world_entry< /// own `0xFF` request does. The answer rides the group event relay onto the same session writer /// and so lands after the list. A failure costs only the marks, so it logs. fn request_target_icons(store: &St, character: Actor) { - let Some(realm) = store.realm_store() else { - return; + let realm = match store.realm_store() { + Ok(Some(realm)) => realm, + Ok(None) => return, + Err(error) => { + log::warn!( + "party: cannot request Target Icons for Character {}: {error:#}", + character.guid() + ); + return; + } }; let op = Op::TargetIcon { icon: lyracore_shared::group::TARGET_ICON_LIST_REQUEST, @@ -1782,7 +1790,7 @@ pub(crate) fn sync_arrival_mirror( store: &St, self_guid: u64, ) -> Result> { - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { return Ok(None); }; let roster = realm @@ -1830,7 +1838,7 @@ pub(crate) fn sync_transfer_arrival_mirror Result<()> { - let Some(realm) = store.party_cleanup_realm()? else { + let Some(realm) = store.realm_store()? else { return Ok(()); }; let roster = realm diff --git a/gateway/src/world/party_mirror.rs b/gateway/src/world/party_mirror.rs index 59bb203f..6dd315c9 100644 --- a/gateway/src/world/party_mirror.rs +++ b/gateway/src/world/party_mirror.rs @@ -70,7 +70,7 @@ impl RosterRevisionRelay { if parties.is_empty() { return; } - let realm = match store.party_cleanup_realm() { + let realm = match store.realm_store() { Ok(Some(realm)) => realm, Ok(None) => return, Err(error) => { diff --git a/gateway/src/world/party_tests.rs b/gateway/src/world/party_tests.rs index 238bf189..7cd609a6 100644 --- a/gateway/src/world/party_tests.rs +++ b/gateway/src/world/party_tests.rs @@ -322,7 +322,7 @@ fn configured_realm_core_unavailability_fails_closed() { ..Default::default() }, topology: TopologyState { - party_command_realm_error: Some("Realm-core unavailable".into()), + realm_error: Some("Realm-core unavailable".into()).into(), ..Default::default() }, ..Default::default() @@ -1110,7 +1110,7 @@ fn an_unavailable_realm_core_preserves_party_membership() { shard: "world".into(), calls, realm: Some(realm.clone()), - party_cleanup_realm_error: Some("Realm-core is not connected".into()), + realm_error: Some("Realm-core is not connected".into()).into(), ..Default::default() }, ..Default::default() @@ -1123,6 +1123,32 @@ fn an_unavailable_realm_core_preserves_party_membership() { assert!(realm.group_roster(VIM).unwrap().is_some()); } +#[test] +fn a_party_leave_during_realm_core_loss_preserves_authority_and_mirrors() { + let (realm, world, instances, _calls) = party_topology(); + form_split_party(&world, &instances); + let before = realm.group_roster(VIM).unwrap().unwrap(); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + let error = party::run( + world.as_ref(), + Actor::new(GINGER).unwrap(), + party::Op::Leave, + ) + .expect_err("a World Shard cannot become party authority"); + + assert_eq!( + crate::stdb::classify(&error), + crate::stdb::DurableFailure::TransportLoss + ); + for shard in [&realm, &world, &instances] { + assert_eq!( + shard.group_roster(VIM).unwrap().unwrap().member_guids(), + before.member_guids() + ); + } +} + #[test] fn repeated_deleted_character_cleanup_is_harmless() { let (realm, world, _instances, _calls) = topology_after_vim_is_deleted(); @@ -1427,7 +1453,7 @@ fn an_unsharded_gateway_runs_every_party_op_on_the_players_own_shard() { ..Default::default() // no `realm`, no `peers` — the unconfigured gateway }); assert!( - store.realm_store().is_none(), + store.realm_store().unwrap().is_none(), "an unsharded store must not name a realm database" ); @@ -2686,7 +2712,7 @@ fn an_unsharded_deployment_still_routes_a_bot_invite_through_realm_group_op() { ..Default::default() // no `realm`, no `peers` — the unconfigured gateway }); assert!( - store.realm_store().is_none(), + store.realm_store().unwrap().is_none(), "an unsharded store must not name a realm database" ); diff --git a/gateway/src/world/presence.rs b/gateway/src/world/presence.rs index c4752c03..bf555f76 100644 --- a/gateway/src/world/presence.rs +++ b/gateway/src/world/presence.rs @@ -199,7 +199,7 @@ pub(crate) fn of( let identity = identity.unwrap_or_else(|| CharacterIdentity::unknown(guid)); return Ok(Some(identity.in_world(shard_name, entity))); } - if realm_transfer_pending(store, guid)? { + if realm_transfer_pending(store, guid).map_err(PresenceUnknown)? { let identity = identity.unwrap_or_else(|| CharacterIdentity::unknown(guid)); return Ok(Some(identity.stationary(Whereabouts::InTransit))); } @@ -224,9 +224,9 @@ pub(crate) fn of( }) } -/// A negative claim [`of`] could not make: some configured World Shard did not vouch for the -/// absence. That is another Shard's health, not this World Session's transport, so a caller answers -/// as it would for a Character it cannot see. +/// A negative claim [`of`] could not make: a configured World Shard did not vouch for absence, +/// or Realm-core could not answer whether a Transfer is pending. A caller answers as it would +/// for a Character it cannot see, without ending this World Session. #[derive(Debug)] pub(crate) struct PresenceUnknown(pub(crate) anyhow::Error); @@ -294,9 +294,9 @@ fn live_entity_anywhere( } /// Realm-core's own pending-Transfer signal for `guid`. `false` on an unsharded Gateway, which has -/// no Realm-core to ask (`realm_store()` answers `None`). +/// no Realm-core to ask (`realm_store()` answers `Ok(None)`). fn realm_transfer_pending(store: &St, guid: u64) -> Result { - let Some(realm) = store.realm_store() else { + let Some(realm) = store.realm_store()? else { return Ok(false); }; Ok(realm diff --git a/gateway/src/world/presence_tests.rs b/gateway/src/world/presence_tests.rs index 7d83c7f2..addc164d 100644 --- a/gateway/src/world/presence_tests.rs +++ b/gateway/src/world/presence_tests.rs @@ -219,6 +219,36 @@ fn a_pending_transfer_on_realm_core_is_in_transit() { assert_eq!(presence.whereabouts, presence::Whereabouts::InTransit); } +#[test] +fn realm_core_loss_leaves_negative_whereabouts_unknown() { + let (_realm, world) = one_shard_topology(GINGER, false); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + for guid in [GINGER, 99_999] { + let error = presence::of(world.as_ref(), guid) + .expect_err("an unreadable pending Transfer cannot become absence"); + assert!(presence::is_unknown(&error)); + assert!( + presence::unknown_as_absent::(Err(error)) + .unwrap() + .is_none() + ); + } +} + +#[test] +fn a_live_entity_still_answers_presence_during_realm_core_loss() { + let (_realm, world) = one_shard_topology(GINGER, true); + *world.topology.realm_error.lock().unwrap() = Some("Realm-core unavailable".into()); + + let presence = presence::of(world.as_ref(), GINGER).unwrap().unwrap(); + + assert!(matches!( + presence.whereabouts, + presence::Whereabouts::InWorld { .. } + )); +} + /// A Character between places on any connected Shard is in transit — the session-less bot /// Transfer Intent case `character_in_transit` models, here on a PEER Shard rather than the /// asking one. diff --git a/gateway/src/world/store.rs b/gateway/src/world/store.rs index 8e02a856..e242c06e 100644 --- a/gateway/src/world/store.rs +++ b/gateway/src/world/store.rs @@ -127,23 +127,10 @@ pub(crate) trait ShardRoutingStore: Send + Sync { /// already did it. Called at world entry whenever the session's home shard is not the realm. fn bind_shard_session(&self, account_id: u64, session_key: &[u8; 40]) -> Result<()>; - /// The **realm-core** handle: the database that owns party membership realm-wide. - /// - /// `None` is not "no realm-core configured" — it is "this gateway runs against ONE database", in - /// which case that database already is the authority and there is nothing to route. `world::party` - /// branches on exactly this, so the single-database path never reads a row it did not read before. - fn realm_store(&self) -> Option>; - - /// Realm-core for deleted Character cleanup. An unavailable configured Realm-core is an - /// infrastructure failure and cannot fall back to a World Shard. - fn party_cleanup_realm(&self) -> Result>>; - - /// Realm-core for companion command authority. Configured outages fail closed. - fn party_command_realm(&self) -> Result>>; - - /// Realm-core for Transfer locator authority. Only an unsharded Realm uses the local Store; - /// an unavailable configured Realm-core is an infrastructure failure. - fn transfer_realm(&self) -> Result>>; + /// Realm-core authority, or `Ok(None)` when this Realm has one Shard and this Store owns it. + /// An unavailable configured Realm-core is an infrastructure failure. Callers must never + /// substitute a World Shard for that authority. + fn realm_store(&self) -> Result>>; /// Every connected WORLD shard's handle (realm-core excluded — it owns no gameplay reads). The /// fan-out set for the roster mirror; empty on a single-database gateway, which is what makes the diff --git a/gateway/src/world/test_support/topology.rs b/gateway/src/world/test_support/topology.rs index aff66070..e34e1b6d 100644 --- a/gateway/src/world/test_support/topology.rs +++ b/gateway/src/world/test_support/topology.rs @@ -28,10 +28,8 @@ pub(crate) struct TopologyState { /// SHARED between a store and its `home` handle: `(shard, call)` for every instrumented /// player-scoped call, in order. The routing test asserts nothing lands on the wrong database. pub(crate) calls: std::sync::Arc>>, - /// The handle `realm_store()` hands back — the database that owns party - /// membership realm-wide. `None` (derive-Default) is the SINGLE-DATABASE gateway, which is what - /// every other test in this file is, and it is what routes every party op back onto the - /// player-facing reducers below. + /// Realm-core authority when configured. `None` models a single-Shard Realm; `realm_error` + /// models an outage without changing which Store owns the realm-wide state. pub(crate) realm: Option>, /// The connected WORLD shards `world_stores()` fans the roster mirror out to (and the /// cross-shard name/presence lookups walk). Empty = single database. Behind a `Mutex` only so @@ -40,10 +38,8 @@ pub(crate) struct TopologyState { pub(crate) peers: std::sync::Mutex>>, /// When set, the configured World Shard set is incomplete or unhealthy. pub(crate) world_shard_set_error: Option, - /// When set, deleted Character cleanup cannot reach Realm-core. - pub(crate) party_cleanup_realm_error: Option, - pub(crate) party_command_realm_error: Option, - pub(crate) transfer_realm_error: Option, + /// A configured Realm-core outage, mutable so a recovery test can restore the same authority. + pub(crate) realm_error: std::sync::Mutex>, /// When set, `settle_home_shard` fails with this message (a transfer that could not be /// driven — an unreachable destination shard, a refused import). pub(crate) settle_error: Option, @@ -109,32 +105,15 @@ impl ShardRoutingStore for WorldFake { (*m == map_id && *i == instance_id).then(|| shard.clone() as std::sync::Arc) } - fn realm_store(&self) -> Option> { - self.topology - .realm - .clone() - .map(|r| r as std::sync::Arc) - } - - fn party_cleanup_realm(&self) -> Result>> { - if let Some(error) = &self.topology.party_cleanup_realm_error { - return Err(anyhow!(error.clone())); - } - Ok(self.realm_store()) - } - - fn party_command_realm(&self) -> Result>> { - if let Some(error) = &self.topology.party_command_realm_error { - return Err(anyhow!(error.clone())); - } - Ok(self.realm_store()) - } - - fn transfer_realm(&self) -> Result>> { - if let Some(error) = &self.topology.transfer_realm_error { + fn realm_store(&self) -> Result>> { + if let Some(error) = self.topology.realm_error.lock().unwrap().as_ref() { return Err(anyhow!(error.clone())); } - Ok(self.realm_store()) + Ok(self + .topology + .realm + .clone() + .map(|r| r as std::sync::Arc)) } fn world_stores(&self) -> Vec> { diff --git a/gateway/src/world/transfer.rs b/gateway/src/world/transfer.rs index 54f45a82..03218eb1 100644 --- a/gateway/src/world/transfer.rs +++ b/gateway/src/world/transfer.rs @@ -824,7 +824,7 @@ fn bind_bot_locator( if intent.source_locator_revision != 0 { return Ok(intent.clone()); } - let realm = holder.transfer_realm()?; + let realm = holder.realm_store()?; let source_revision = realm .as_deref() .unwrap_or(holder) diff --git a/gateway/src/world/transfer_tests.rs b/gateway/src/world/transfer_tests.rs index bafd6d77..74ea076b 100644 --- a/gateway/src/world/transfer_tests.rs +++ b/gateway/src/world/transfer_tests.rs @@ -1522,7 +1522,8 @@ fn a_ready_intent_keeps_retrying_when_the_local_character_is_elsewhere() { fn an_unbound_intent_waits_for_configured_realm_core() { let (src, src_db, dst_db, calls) = bot_pair(36, 7); let mut holder = std::sync::Arc::try_unwrap(src).ok().unwrap(); - holder.topology.transfer_realm_error = Some("configured Realm-core is unavailable".into()); + *holder.topology.realm_error.get_mut().unwrap() = + Some("configured Realm-core is unavailable".into()); let intent = super::transfer::BotTransferIntent { source_locator_revision: 0, ..bot_intent()