From 7c2d3b734c6371dd98c318937156c291be293754 Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Thu, 3 Sep 2026 22:47:40 +0200 Subject: [PATCH 1/9] test(gateway): add an in-memory relay scaling harness A synthetic realm of N viewers, 4N creatures and 5N auras, no sockets and no database, that times a burst of combat, cast, impact and aura relay work at 500, 1000 and 2000 players. Ignored by default; run it with --ignored --nocapture in release. Co-Authored-By: Claude Fable 5.1 --- gateway/src/stdb/world_view.rs | 3 + gateway/src/stdb/world_view/relay_bench.rs | 350 +++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 gateway/src/stdb/world_view/relay_bench.rs diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index d69f36f82..e4810f85c 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -2019,6 +2019,9 @@ fn duel_winner_audience(viewer_guid: u64, initiator_guid: u64, challenged_guid: viewer_guid != 0 && viewer_guid != initiator_guid && viewer_guid != challenged_guid } +#[cfg(test)] +mod relay_bench; + #[cfg(test)] mod family_audience_tests { use super::{ diff --git a/gateway/src/stdb/world_view/relay_bench.rs b/gateway/src/stdb/world_view/relay_bench.rs new file mode 100644 index 000000000..da3291d72 --- /dev/null +++ b/gateway/src/stdb/world_view/relay_bench.rs @@ -0,0 +1,350 @@ +//! Relay scaling harness: a synthetic realm in memory, no sockets, no database. +//! +//! Run with `cargo test -p lyracore-gateway --release relay_scaling -- --ignored --nocapture`. +//! It prints one row per population size: the pump cost of a burst of combat, cast and impact +//! events, the writer-side cost of draining every job those enqueued, and the aura relay's +//! audience and array-sync cost. Ignored by default because it is a measurement, not a check. + +use super::*; +use crate::codec::property_tests::Rng; +use crate::stdb::bindings::{Aura, CombatEvent, SpellCastEvent, SpellImpactEvent}; +use crate::world::{Outbound, SessionTx}; +use std::sync::mpsc::Receiver; +use std::time::{Duration, Instant}; + +const PLAYERS_PER_CELL: usize = 8; +const CREATURES_PER_PLAYER: usize = 4; +const AURAS_PER_PLAYER: usize = 5; +const AURA_BURST: usize = 50; +const PLAYER_GUID_BASE: u64 = 0x0000_0000_0001_0000; +const CREATURE_GUID_BASE: u64 = 0xF130_0000_0001_0000; + +struct SyntheticRealm { + view: Arc, + viewers: Vec>, + queues: Vec>, + creatures: Vec<(u64, CellKey)>, + auras: Vec, +} + +fn viewer(session: SessionId, self_guid: u64, tx: SessionTx) -> Arc { + let mut identity = [0u8; 32]; + identity[..8].copy_from_slice(&self_guid.to_le_bytes()); + Arc::new(Viewer { + session, + self_guid, + bound_identity: spacetimedb_sdk::Identity::from_byte_array(identity), + map_id: 0, + instance_id: 0, + zone_id: 0.into(), + tx, + created: Arc::new(Mutex::new(HashSet::from([self_guid]))), + gates: Arc::new(ViewerGates::default()), + skill_slots: Arc::new(Mutex::new((HashMap::new(), 0))), + explored: Mutex::new(ExplorationReplay::default()), + motion_pending: Arc::new(MotionPending::default()), + }) +} + +fn aura(id: u64, target_guid: u64, eff_kind: u8) -> Aura { + Aura { + id, + target_guid, + caster_guid: target_guid, + spell_id: 1000 + (id % 40) as u32, + slot: (id % 32) as u8, + level: 60, + flags: 0x1f, + applied_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + expires_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + effect_id: id, + eff_kind, + amount: 10, + eff_p0: 0, + eff_p0_kind: 0, + eff_p1: 0, + period_ms: 0, + amount_remaining: 0, + stacks: 1, + next_tick_micros: 0, + channel_target: 0, + enters_combat: false, + proc_flags: 0, + proc_chance: 0, + proc_ppm: 0.0, + proc_ex: 0, + proc_school_mask: 0, + proc_family_name: 0, + proc_family_flags: 0, + proc_charges: 0, + proc_icd_ms: 0, + proc_ready_micros: 0, + } +} + +/// `players` viewers and `4 x players` creatures spread over a square of cells at eight players +/// per cell, every viewer's `created` set seeded with what its box can see, and five auras per +/// player spread over every entity. +fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { + let side = ((players / PLAYERS_PER_CELL) as f64).sqrt().ceil() as i32; + let cell = |rng: &mut Rng| CellKey::at(0, 0, rng.below(side as usize) as i32, rng.below(side as usize) as i32); + let view = Arc::new(WorldView::new(true)); + let mut viewers = Vec::with_capacity(players); + let mut queues = Vec::with_capacity(players); + for i in 0..players { + let guid = PLAYER_GUID_BASE + i as u64; + let (tx, rx) = SessionTx::with_depth(0); + let v = viewer(view.next_session_id(), guid, tx); + let anchor = cell(rng); + view.spatial + .upsert_entity(EntityLayer::WorldEntity, guid, anchor, 0); + view.add_viewer_on_shard(v.clone(), anchor, 0); + viewers.push(v); + queues.push(rx); + } + let creatures: Vec<(u64, CellKey)> = (0..players * CREATURES_PER_PLAYER) + .map(|i| (CREATURE_GUID_BASE + i as u64, cell(rng))) + .collect(); + for (guid, key) in &creatures { + view.spatial + .upsert_entity(EntityLayer::WorldEntity, *guid, *key, 0); + } + for v in &viewers { + let visible = view + .spatial + .visible_entities(EntityLayer::WorldEntity, v.session); + v.created + .lock() + .unwrap() + .extend(visible.into_iter().map(|(guid, _)| guid)); + } + let entities = players + creatures.len(); + let auras: Vec = (0..players * AURAS_PER_PLAYER) + .map(|i| { + let pick = rng.below(entities); + let target = if pick < players { + PLAYER_GUID_BASE + pick as u64 + } else { + creatures[pick - players].0 + }; + let kind = if i % 20 == 0 { super::super::subscriptions::A_STEALTH } else { 0xA0 }; + aura(1 + i as u64, target, kind) + }) + .collect(); + SyntheticRealm { + view, + viewers, + queues, + creatures, + auras, + } +} + +impl SyntheticRealm { + fn player_cell(&self, i: usize) -> CellKey { + self.view + .spatial + .entity_cell(EntityLayer::WorldEntity, self.viewers[i].self_guid) + .expect("every synthetic player is indexed") + } + + fn combat_row(&self, attacker: usize, rng: &mut Rng) -> CombatEvent { + let key = self.player_cell(attacker); + let (gx, gy) = lyracore_shared::spatial::grid_cell_of_id(key.cell); + CombatEvent { + id: attacker as u64, + attacker_guid: self.viewers[attacker].self_guid, + target_guid: self.creatures[rng.below(self.creatures.len())].0, + damage: 17, + hit_info: 0, + killing_blow: false, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + blocked_amount: 0, + ranged_spell_id: 0, + ammo_display_id: 0, + spell_swing: false, + impact_delay_ms: 0, + map_id: 0, + instance_id: 0, + grid_x: gx, + grid_y: gy, + } + } + + fn cast_row(&self, caster: usize, rng: &mut Rng) -> SpellCastEvent { + let key = self.player_cell(caster); + let (gx, gy) = lyracore_shared::spatial::grid_cell_of_id(key.cell); + SpellCastEvent { + id: caster as u64, + caster_guid: self.viewers[caster].self_guid, + spell_id: 133, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + target_guid: self.creatures[rng.below(self.creatures.len())].0, + cast_time_ms: 0, + is_completion: true, + damage: 40, + school: 2, + is_crit: false, + resisted: 0, + absorbed: 0, + is_interrupted: false, + cooldown_ms: 0, + delay_ms: 0, + healed: 0, + is_proc_log: false, + swing_hit_info: 0, + client_initiated: false, + map_id: 0, + instance_id: 0, + grid_x: gx, + grid_y: gy, + failure_reason: 0, + } + } + + fn impact_row(&self, caster: usize, rng: &mut Rng) -> SpellImpactEvent { + let key = self.player_cell(caster); + let (gx, gy) = lyracore_shared::spatial::grid_cell_of_id(key.cell); + SpellImpactEvent { + id: caster as u64, + caster_guid: self.viewers[caster].self_guid, + target_guid: self.creatures[rng.below(self.creatures.len())].0, + spell_id: 116, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + damage: 33, + school: 4, + is_crit: false, + resisted: 0, + absorbed: 0, + map_id: 0, + instance_id: 0, + grid_x: gx, + grid_y: gy, + } + } + + /// Run every queued job on every writer queue: `(jobs run, packets produced)`. + fn drain(&self) -> (usize, usize) { + let (mut jobs, mut packets) = (0, 0); + for rx in &self.queues { + while let Ok(out) = rx.try_recv() { + match out { + Outbound::Job(job) => { + jobs += 1; + packets += job().len(); + } + _ => packets += 1, + } + } + } + (jobs, packets) + } +} + +struct Burst { + pump: Duration, + drain: Duration, + jobs: usize, + packets: usize, +} + +fn burst(realm: &SyntheticRealm, dispatch: impl Fn(usize)) -> Burst { + let started = Instant::now(); + for i in 0..realm.viewers.len() { + dispatch(i); + } + let pump = started.elapsed(); + let started = Instant::now(); + let (jobs, packets) = realm.drain(); + Burst { + pump, + drain: started.elapsed(), + jobs, + packets, + } +} + +fn micros_per(total: Duration, count: usize) -> f64 { + total.as_secs_f64() * 1e6 / count as f64 +} + +/// The aura relay, event by event: audience selection on the pump, then the per-recipient job +/// gate and the aura array sync each surviving job builds. +fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Duration, usize) { + let mut audience = Duration::ZERO; + let mut sync = Duration::ZERO; + let (mut candidates, mut synced) = (0, 0); + for _ in 0..AURA_BURST { + let row = &realm.auras[rng.below(realm.auras.len())]; + let started = Instant::now(); + let recipients = realm.view.viewers_on_shard(0); + audience += started.elapsed(); + candidates += recipients.len(); + for viewer in recipients { + let visible = row.target_guid == viewer.self_guid + || viewer.created.lock().unwrap().contains(&row.target_guid); + if !visible { + continue; + } + let started = Instant::now(); + let _ = super::super::subscriptions::aura_sync( + realm.auras.iter().cloned(), + row.target_guid, + ); + sync += started.elapsed(); + synced += 1; + } + } + (audience, candidates, sync, synced) +} + +#[test] +#[ignore] +fn relay_scaling() { + println!(); + println!( + "| players | creatures | auras | relay | pump us/event | jobs/event | drain us/event | packets/event |" + ); + println!("|---|---|---|---|---|---|---|---|"); + for players in [500usize, 1000, 2000] { + let mut rng = Rng::new(0x5CA1_AB1E_0000_0001 + players as u64); + let realm = build(players, &mut rng); + let n = players; + let combat = { + let rows: Vec = (0..n).map(|i| realm.combat_row(i, &mut rng)).collect(); + burst(&realm, |i| combat_event_appeared(&realm.view, 0, &rows[i])) + }; + let cast = { + let rows: Vec = (0..n).map(|i| realm.cast_row(i, &mut rng)).collect(); + burst(&realm, |i| cast_event_appeared(&realm.view, 0, &rows[i])) + }; + let impact = { + let rows: Vec = + (0..n).map(|i| realm.impact_row(i, &mut rng)).collect(); + burst(&realm, |i| impact_appeared(&realm.view, 0, &rows[i])) + }; + for (name, b) in [("combat", combat), ("cast", cast), ("impact", impact)] { + println!( + "| {players} | {} | {} | {name} | {:.1} | {:.0} | {:.1} | {:.1} |", + realm.creatures.len(), + realm.auras.len(), + micros_per(b.pump, n), + b.jobs as f64 / n as f64, + micros_per(b.drain, n), + b.packets as f64 / n as f64, + ); + } + let (audience, candidates, sync, synced) = aura_burst(&realm, &mut rng); + println!( + "| {players} | {} | {} | aura | {:.1} | {:.0} | {:.1} | {:.1} |", + realm.creatures.len(), + realm.auras.len(), + micros_per(audience, AURA_BURST), + candidates as f64 / AURA_BURST as f64, + micros_per(sync, AURA_BURST), + synced as f64 / AURA_BURST as f64, + ); + } + println!(); + println!("aura columns: pump = audience selection, jobs = candidate viewers, drain = array syncs built, packets = viewers past the visibility gate"); +} From e42f31dd7ee8c562340d7d4a2ef51c2b8f8425bb Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Thu, 3 Sep 2026 23:02:43 +0200 Subject: [PATCH 2/9] perf(gateway): route event relays through the cell index Combat, melee stance, cast, impact, emote, chat and aura rows enqueued one job per viewer on the shard and let the job's gate reject the row, so each event cost the shard's population and the total grew with its square. The Module already stamps the actor's cell on the event rows; the gateway now anchors each relay on that cell (or the actor's indexed cell) and asks the cell index for the viewers whose box covers it. The per-viewer gate stays the final filter. A yell widens the span to its 300 yd range. The aura cache is indexed by target on the gateway, keyed by (shard, id), so the array sync, the run-speed fold, the stealth count on the pump and the peer CREATE's stealth check read one unit's auras instead of scanning the shard. Co-Authored-By: Claude Fable 5.1 --- docs/architecture.md | 6 +- gateway/src/stdb/subscriptions.rs | 115 +-- gateway/src/stdb/world_index.rs | 56 +- gateway/src/stdb/world_view.rs | 372 +++++++--- .../stdb/world_view/cell_audience_tests.rs | 686 ++++++++++++++++++ gateway/src/stdb/world_view/relay_bench.rs | 63 +- 6 files changed, 1141 insertions(+), 157 deletions(-) create mode 100644 gateway/src/stdb/world_view/cell_audience_tests.rs diff --git a/docs/architecture.md b/docs/architecture.md index b0f858e98..cc1e57e71 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -358,7 +358,11 @@ Every relay hangs off a coordinator connection, in one of two shapes: families (entities, motion, combat, chat, auras, corpses, casts), the recipient-keyed PRIVATE tier (whisper/group/resurrect), and owner-addressed XP, level-up, exploration, quest, item, teleport, addon, and reputation rows. GUID and bound-identity indexes select one viewer directly; - the callback enqueues packet work on that session's FIFO writer. The cross-shard whisper/group + the callback enqueues packet work on that session's FIFO writer. Combat, melee stance, cast, + impact, emote, chat and aura rows are cell-anchored: the Module stamps the actor's cell on the + event row (an aura or chat line uses the actor's indexed cell), the shared cell index picks the + viewers whose box covers it, and the job's per-viewer gate stays the final filter. Only rolls, + corpses, dynamic objects, channel lines and weather still fan out per shard. The cross-shard whisper/group twins ride the same dispatchers on the realm-core connection (`arm_realm_private`), armed only when realm-core is a distinct database. - **Viewer lifetime** (`subscribe_player_events`): world entry prepares relay state, registers one diff --git a/gateway/src/stdb/subscriptions.rs b/gateway/src/stdb/subscriptions.rs index 78eba8a66..35cf7a9c3 100644 --- a/gateway/src/stdb/subscriptions.rs +++ b/gateway/src/stdb/subscriptions.rs @@ -704,10 +704,25 @@ fn peer_create_gate( /// expects of its callers). Module-level (not local to `on_chat`) so the range constants live /// beside the discriminant they gate. const CHAT_YELL: u8 = 1; -/// SAY proximity range, squared (vanilla ~25yd). See [`chat_in_range`]. -const SAY_RANGE_SQ: f32 = 25.0 * 25.0; // 625.0 yd² -/// YELL proximity range, squared (vanilla ~300yd). Same rationale as `SAY_RANGE_SQ`. -const YELL_RANGE_SQ: f32 = 300.0 * 300.0; // 90_000.0 yd² +/// SAY proximity range (vanilla ~25yd). See [`chat_in_range`]. +const SAY_RANGE_YD: f32 = 25.0; +/// YELL proximity range (vanilla ~300yd). Same rationale as `SAY_RANGE_YD`. +const YELL_RANGE_YD: f32 = 300.0; +#[cfg(test)] +const SAY_RANGE_SQ: f32 = SAY_RANGE_YD * SAY_RANGE_YD; // 625.0 yd² +#[cfg(test)] +const YELL_RANGE_SQ: f32 = YELL_RANGE_YD * YELL_RANGE_YD; // 90_000.0 yd² + +/// How far a `game_chat_event` line of this kind carries. Every kind on that table is +/// range-bound: SAY and text emotes at SAY range, YELL at its own; party, guild and whispers +/// ride other tables. +pub(crate) fn chat_range_yd(chat_type: u8) -> f32 { + if chat_type == CHAT_YELL { + YELL_RANGE_YD + } else { + SAY_RANGE_YD + } +} // ================================================================================================== // The SHARED-dispatch relay bodies. @@ -738,6 +753,7 @@ const YELL_RANGE_SQ: f32 = 300.0 * 300.0; // 90_000.0 yd² shared-dispatch path's differential test exists to prevent"] pub(crate) fn offer_peer_create_for( coord: &Coordinator, + view: &WorldView, viewer: &Viewer, row: &WorldEntity, ) -> Vec { @@ -746,18 +762,13 @@ pub(crate) fn offer_peer_create_for( return Vec::new(); } let viewer_is_ghost = viewer.gates.is_ghost(); - // Short-circuit BEFORE the aura scan: a spirit healer viewed by a living player is refused - // regardless of stealth, and the code before the shared-connection model returned here without - // ever touching game_aura. + // A spirit healer viewed by a living player is refused regardless of stealth. if row.npc_flags & SPIRITHEALER_NPC_FLAG != 0 && !viewer_is_ghost { return Vec::new(); } + let row_is_stealthed = view.auras.is_stealthed(row.guid); let guard = coord.0.coord(); let db = &guard.conn.db; - let row_is_stealthed = db - .game_aura() - .iter() - .any(|a| a.target_guid == row.guid && a.eff_kind == A_STEALTH); if !peer_create_gate( row.guid, viewer.self_guid, @@ -861,6 +872,7 @@ fn append_encounter_equip_after_create( shared-dispatch path's differential test exists to prevent"] pub(crate) fn relay_entity_update( coord: &Coordinator, + view: &WorldView, viewer: &Viewer, old: &WorldEntity, new: &WorldEntity, @@ -870,7 +882,7 @@ pub(crate) fn relay_entity_update( if new.guid != viewer.self_guid { let shown = viewer.created.lock().unwrap().contains(&new.guid); if is_update_reentry(new.guid, viewer.self_guid, shown) { - return offer_peer_create_for(coord, viewer, new); + return offer_peer_create_for(coord, view, viewer, new); } } let dynamic_flags = { @@ -906,7 +918,7 @@ pub(crate) fn relay_entity_update( drop(guard); for h in healers { if is_ghost { - out.extend(offer_peer_create_for(coord, viewer, &h)); + out.extend(offer_peer_create_for(coord, view, viewer, &h)); } else if viewer.created.lock().unwrap().remove(&h.guid) { out.push(Outbound::One(ServerOpcodeMessage::SMSG_DESTROY_OBJECT( codec::build_destroy_object(h.guid), @@ -1445,6 +1457,9 @@ pub(crate) fn cast_event_outbound(self_guid: u64, row: &SpellCastEvent) -> Vec, target_guid: u64, @@ -1525,10 +1540,10 @@ pub(crate) fn sheet_packet(coord: &Coordinator, changed: &Aura, self_guid: u64) } // Stealth peer-visibility: when a NON-self peer's A_STEALTH presence crosses the 0↔1 boundary, // HIDE it from this viewer (SMSG_DESTROY_OBJECT + evict from `created`) on the gain, REVEAL it -// (re-CREATE + re-insert into `created`) on the loss. The recipient set is implicit — every -// viewer's own connection drains the broadcast `game_aura` table and runs THIS closure, so each -// in-scope client hides/reveals on its own `tx`. Self is excluded (`changed.target_guid != -// self_guid`) so a stealther never hides from itself. Idempotency is the `created` set: HIDE only +// (re-CREATE + re-insert into `created`) on the loss. The dispatch runs this once per viewer +// whose box covers the stealther, so each in-scope client hides/reveals on its own `tx`. Self is +// excluded (`changed.target_guid != self_guid`) so a stealther never hides from itself. +// Idempotency is the `created` set: HIDE only // fires (and DESTROYs) if the guid was created; REVEAL only fires (and CREATEs) if it wasn't — // re-hiding a hidden peer or re-revealing a visible one is a no-op. The stealther's entity row is // read from the firing connection's cache (`ctx.db`); a guid the viewer can't see in scope has no @@ -1610,11 +1625,9 @@ pub(crate) fn stealth_visibility( } } -/// Aura insert leg — the shared-dispatch twin of the per-player -/// `on_aura_insert` closure. Same helper stack (`aura_sync`/`aura_duration_packet`/ -/// `run_speed_packet`/`armor_packet`/`stealth_visibility`); the aura iterators read the -/// COORDINATOR cache (identical rows — both subscriptions were `SELECT *`), and -/// `stealth_count` was computed on the coordinator pump, where that cache is exactly post-change. +/// Aura insert leg: array sync, self-only duration/run-speed/armor/sheet packets, and the +/// stealth HIDE transition. The target's current aura set comes from the gateway's aura index, +/// post-change like the cache it mirrors; `stealth_count` was taken on the pump. #[allow(clippy::too_many_arguments)] pub(crate) fn aura_insert_outbound( coord: &Coordinator, @@ -1626,22 +1639,19 @@ pub(crate) fn aura_insert_outbound( stealth_count: usize, ) -> Vec { let mut out = Vec::new(); - // See the per-player twin: a stealth-hidden peer (not in `created`) must get NO per-peer - // relay — a partial VALUES on a DESTROYed object is a client crash/desync vector. + // A stealth-hidden peer (not in `created`) must get NO per-peer relay: a partial VALUES on + // a DESTROYed object is a client crash/desync vector. let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); - if visible { - let guard = coord.0.coord(); - out.push(aura_sync(guard.conn.db.game_aura().iter(), row.target_guid)); + let current = visible.then(|| view.auras.on_target(row.target_guid)); + if let Some(current) = ¤t { + out.push(aura_sync(current.iter().cloned(), row.target_guid)); } if let Some(o) = aura_duration_packet(row, self_guid) { out.push(o); } - if visible { - { - let guard = coord.0.coord(); - if let Some(o) = run_speed_packet(guard.conn.db.game_aura().iter(), row, self_guid) { - out.push(o); - } + if let Some(current) = current { + if let Some(o) = run_speed_packet(current.into_iter(), row, self_guid) { + out.push(o); } if let Some(o) = armor_packet(coord, row, self_guid) { out.push(o); @@ -1663,9 +1673,10 @@ pub(crate) fn aura_insert_outbound( out } -/// Aura update leg — twin of `on_aura_update` (no stealth transition on an update). +/// Aura update leg (no stealth transition on an update). pub(crate) fn aura_update_outbound( coord: &Coordinator, + view: &WorldView, created: &Arc>>, self_guid: u64, row: &Aura, @@ -1673,9 +1684,9 @@ pub(crate) fn aura_update_outbound( ) -> Vec { let mut out = Vec::new(); let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); - if visible { - let guard = coord.0.coord(); - out.push(aura_sync(guard.conn.db.game_aura().iter(), row.target_guid)); + let current = visible.then(|| view.auras.on_target(row.target_guid)); + if let Some(current) = ¤t { + out.push(aura_sync(current.iter().cloned(), row.target_guid)); } // Re-send the timer only when the duration window actually changed (a refresh). if expires_changed { @@ -1683,12 +1694,9 @@ pub(crate) fn aura_update_outbound( out.push(o); } } - if visible { - { - let guard = coord.0.coord(); - if let Some(o) = run_speed_packet(guard.conn.db.game_aura().iter(), row, self_guid) { - out.push(o); - } + if let Some(current) = current { + if let Some(o) = run_speed_packet(current.into_iter(), row, self_guid) { + out.push(o); } if let Some(o) = armor_packet(coord, row, self_guid) { out.push(o); @@ -1700,8 +1708,8 @@ pub(crate) fn aura_update_outbound( out } -/// Aura delete leg — twin of `on_aura_delete` (the REVEAL half of stealth). The coordinator -/// cache is post-delete by callback contract, so the sync/speed/armor folds read the remaining set. +/// Aura delete leg (the REVEAL half of stealth). The aura index is post-delete, so the +/// sync/speed folds read the remaining set. #[allow(clippy::too_many_arguments)] pub(crate) fn aura_delete_outbound( coord: &Coordinator, @@ -1715,12 +1723,10 @@ pub(crate) fn aura_delete_outbound( let mut out = Vec::new(); let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); if visible { - { - let guard = coord.0.coord(); - out.push(aura_sync(guard.conn.db.game_aura().iter(), row.target_guid)); - if let Some(o) = run_speed_packet(guard.conn.db.game_aura().iter(), row, self_guid) { - out.push(o); - } + let current = view.auras.on_target(row.target_guid); + out.push(aura_sync(current.iter().cloned(), row.target_guid)); + if let Some(o) = run_speed_packet(current.into_iter(), row, self_guid) { + out.push(o); } if let Some(o) = armor_packet(coord, row, self_guid) { out.push(o); @@ -2238,11 +2244,8 @@ pub(crate) fn chat_event_outbound( row: &ChatEvent, ) -> Vec { if row.sender_guid != self_guid { - let range_sq = if row.chat_type == CHAT_YELL { - YELL_RANGE_SQ - } else { - SAY_RANGE_SQ // SAY and any future proximity type default to SAY range - }; + let range_yd = chat_range_yd(row.chat_type); + let range_sq = range_yd * range_yd; let guard = coord.0.coord(); let speaker = match guard .conn diff --git a/gateway/src/stdb/world_index.rs b/gateway/src/stdb/world_index.rs index 9b5fcf323..f3c77973f 100644 --- a/gateway/src/stdb/world_index.rs +++ b/gateway/src/stdb/world_index.rs @@ -118,10 +118,16 @@ impl CellKey { /// symmetric — `neighbourhood_is_exactly_the_grid_box_bounds` (below) pins it against /// `GridBox::bounds`, the predicate the deleted subscription SQL was built from. pub fn neighbourhood(&self) -> impl Iterator { + self.neighbourhood_within(BOX_HALF_SPAN) + } + + /// The `(2·half_span+1)`-square of keys centred here, this key included. The box is the + /// `BOX_HALF_SPAN` case; a relay whose range exceeds the box (a yell) widens it. + pub fn neighbourhood_within(&self, half_span: i32) -> impl Iterator { let (gx, gy) = lyracore_shared::spatial::grid_cell_of_id(self.cell); let (map_id, instance_id) = (self.map_id, self.instance_id); - ((gx - BOX_HALF_SPAN)..=(gx + BOX_HALF_SPAN)).flat_map(move |cx| { - ((gy - BOX_HALF_SPAN)..=(gy + BOX_HALF_SPAN)) + ((gx - half_span)..=(gx + half_span)).flat_map(move |cx| { + ((gy - half_span)..=(gy + half_span)) .map(move |cy| CellKey::at(map_id, instance_id, cx, cy)) }) } @@ -278,10 +284,9 @@ impl WorldIndex { Some(key) } - /// (Inspection only — the dispatch never needs it; the differential tests do.) - #[cfg(test)] - /// Where the index believes `guid` is. `None` for a guid it has never seen (a row on a table it - /// does not index, or one deleted already). + /// Where the index believes `guid` is: the anchor of a relay driven by a table with no cell of + /// its own (an aura names its target, a chat line its speaker). `None` for a guid it has never + /// seen (a row on a table it does not index, or one deleted already). pub fn entity_cell(&self, layer: EntityLayer, guid: u64) -> Option { let inner = self.lock(); Self::layer(&inner, layer).entity_cell.get(&guid).map(|(k, _)| *k) @@ -340,6 +345,13 @@ impl WorldIndex { /// both layers use the same viewer anchors. O(25 + V) with cell scoping on; O(total viewers) /// with `LYRACORE_AOI=0`. pub fn viewers_of(&self, _layer: EntityLayer, key: CellKey) -> Vec { + self.viewers_within(key, BOX_HALF_SPAN) + } + + /// Every session anchored within `half_span` cells of `key`. The box (`BOX_HALF_SPAN`) is + /// what [`Self::viewers_of`] asks; a relay whose range exceeds the box widens the span so the + /// yard-range gate in its job still sees every candidate. O((2·half_span+1)² + V). + pub fn viewers_within(&self, key: CellKey, half_span: i32) -> Vec { let inner = self.lock(); if !self.cell_scoped { let partition = key.partition(); @@ -351,7 +363,7 @@ impl WorldIndex { .collect(); } let mut out = Vec::new(); - for anchor in key.neighbourhood() { + for anchor in key.neighbourhood_within(half_span) { if let Some(sessions) = inner.cell_viewers.get(&anchor) { out.extend_from_slice(sessions); } @@ -777,6 +789,36 @@ mod tests { .all(|k| k.map_id == 3 && k.instance_id == 9)); } + /// A widened span names exactly the anchors within that many cells on both axes, and the + /// box span is the ordinary `viewers_of` answer. + #[test] + fn viewers_within_widens_the_box_by_chebyshev_distance() { + let index = WorldIndex::new(true); + let centre = CellKey::at(0, 0, 0, 0); + let mut expected: HashMap> = HashMap::new(); + let mut session = 0; + for gx in -8..=8 { + for gy in -8..=8 { + session += 1; + index.add_viewer(session, CellKey::at(0, 0, gx, gy)); + for span in 0..=8 { + if gx.abs() <= span && gy.abs() <= span { + expected.entry(span).or_default().insert(session); + } + } + } + } + index.add_viewer(session + 1, CellKey::at(0, 1, 0, 0)); // another instance + for span in 0..=8 { + let got: HashSet = index.viewers_within(centre, span).into_iter().collect(); + assert_eq!(got, expected[&span], "span {span}"); + } + assert_eq!( + index.viewers_within(centre, BOX_HALF_SPAN), + index.viewers_of(EntityLayer::WorldEntity, centre) + ); + } + // =========================================================================================== // Bookkeeping // =========================================================================================== diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index e4810f85c..07400fd44 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -42,6 +42,7 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use anyhow::{anyhow, Result}; +use lyracore_shared::spatial::{BOX_HALF_SPAN, GRID_CELL_SIZE}; use spacetimedb_sdk::{Table, TableWithPrimaryKey}; use wow_world_messages::vanilla::WeatherChangeType; @@ -198,11 +199,97 @@ impl ViewerRegistry { } } -/// The gateway-wide shared view: one layered spatial index, the live viewer registry, and the -/// shard table the index's [`ShardId`]s point into. +/// The aura cache indexed by target: every `game_aura` row, keyed by the unit it sits on, fed by +/// the same per-shard callbacks that relay it. Every aura read a relay makes is per target (the +/// array sync, the run-speed fold, the stealth count), and the SDK cache indexes rows by id only, +/// so each read used to scan the whole shard's auras. +/// +/// Rows are identified by `(shard, id)`: `id` is a per-shard autoincrement, so a transfer's late +/// cascade delete on the source shard must not evict the destination shard's row of the same id. +#[derive(Default)] +pub(crate) struct AuraIndex { + by_target: Mutex>>, +} + +impl AuraIndex { + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap>> { + self.by_target.lock().unwrap_or_else(|p| { + log::error!( + "aura index lock poisoned (a prior panic in a critical section) — recovering" + ); + p.into_inner() + }) + } + + /// Record or replace a row. A reconnect's initial apply replays resident rows as inserts, and + /// those land on their own `(shard, id)` instead of doubling. + pub(crate) fn upsert(&self, shard: ShardId, row: &Aura) { + let mut by_target = self.lock(); + let rows = by_target.entry(row.target_guid).or_default(); + match rows.iter_mut().find(|(s, a)| *s == shard && a.id == row.id) { + Some(slot) => slot.1 = row.clone(), + None => rows.push((shard, row.clone())), + } + } + + /// Forget a row. Idempotent. + pub(crate) fn remove(&self, shard: ShardId, row: &Aura) { + let mut by_target = self.lock(); + if let Some(rows) = by_target.get_mut(&row.target_guid) { + rows.retain(|(s, a)| !(*s == shard && a.id == row.id)); + if rows.is_empty() { + by_target.remove(&row.target_guid); + } + } + } + + /// Rebuild from the shard caches. The reconnect path: a fresh subscription's apply repopulates + /// the cache without firing a delete for a row that expired while the connection was down. + pub(crate) fn replace_all(&self, rows: impl IntoIterator) { + let mut fresh: HashMap> = HashMap::new(); + for (shard, row) in rows { + fresh.entry(row.target_guid).or_default().push((shard, row)); + } + *self.lock() = fresh; + } + + /// The auras on one unit, cloned out so no encoding ever runs under the lock. + pub(crate) fn on_target(&self, target_guid: u64) -> Vec { + self.lock() + .get(&target_guid) + .map(|rows| rows.iter().map(|(_, a)| a.clone()).collect()) + .unwrap_or_default() + } + + /// How many `A_STEALTH` auras the unit carries right now. + pub(crate) fn stealth_count(&self, target_guid: u64) -> usize { + self.lock() + .get(&target_guid) + .map(|rows| { + rows.iter() + .filter(|(_, a)| a.eff_kind == super::subscriptions::A_STEALTH) + .count() + }) + .unwrap_or(0) + } + + pub(crate) fn is_stealthed(&self, target_guid: u64) -> bool { + self.stealth_count(target_guid) > 0 + } + + /// Live sizes, for the seed log line: `(auras, units carrying one)`. + pub(crate) fn stats(&self) -> (usize, usize) { + let by_target = self.lock(); + (by_target.values().map(Vec::len).sum(), by_target.len()) + } +} + +/// The gateway-wide shared view: one layered spatial index, the aura index, the live viewer +/// registry, and the shard table the index's [`ShardId`]s point into. pub(crate) struct WorldView { /// One authoritative viewer anchor with independent world-entity and game-object layers. pub(crate) spatial: WorldIndex, + pub(crate) auras: AuraIndex, viewers: RwLock, /// [`ShardId`] → that shard's coordinator handle. Written once at startup, read on every /// dispatch; the order fixes the ids, so it must never be reordered after arming. @@ -214,6 +301,7 @@ impl WorldView { pub(crate) fn new(cell_scoped: bool) -> Self { Self { spatial: WorldIndex::new(cell_scoped), + auras: AuraIndex::default(), viewers: RwLock::new(ViewerRegistry::default()), shards: RwLock::new(Vec::new()), next_session: AtomicU64::new(1), @@ -348,6 +436,39 @@ impl WorldView { recipients } + /// The viewers a cell-anchored event reaches: every session anchored within `half_span` cells + /// of `key`, plus the sessions of the characters the row names. The owner leg keeps a + /// self-addressed packet (the attacker's ATTACKSTOP, the caster's cast result, a player's own + /// aura timer) reaching its owner while the owner's anchor and the row's cell disagree by a + /// heartbeat, and it is the whole audience when the anchor is unknown (`None`: the actor's + /// row is gone, so no other viewer has it on screen). + /// + /// The per-viewer gate the job runs is the final filter, unchanged; this only stops the pump + /// enqueueing a job for every viewer on the shard so it can reject the row later. + fn cell_audience( + &self, + key: Option, + half_span: i32, + owners: &[u64], + ) -> Vec> { + let mut sessions = key + .map(|key| self.spatial.viewers_within(key, half_span)) + .unwrap_or_default(); + let registry = self.viewers.read().unwrap(); + for owner in owners { + if let Some(session) = registry.session_of_owner.get(owner) { + if !sessions.contains(session) { + sessions.push(*session); + } + } + } + sessions + .into_iter() + .filter_map(|session| registry.by_session.get(&session)) + .map(|registered| registered.viewer.clone()) + .collect() + } + // =========================================================================================== // Arming — one registration per shard, re-run by the coordinator watchdog after a reconnect. // =========================================================================================== @@ -604,19 +725,20 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId }); // ---- game_combat_event -------------------------------------------------------------------- - // Melee/ranged swing log. Shard broadcast; the created-set visibility gate runs inside the - // job (`combat_event_outbound`), per viewer. - wire_insert(db.game_combat_event(), "game_combat_event.insert", &view, move |v, row| { - combat_event_appeared(v, shard, row) + // Melee/ranged swing log, anchored on the attacker's cell; the created-set visibility gate + // runs inside the job (`combat_event_outbound`), per viewer. + wire_insert(db.game_combat_event(), "game_combat_event.insert", &view, |v, row| { + combat_event_appeared(v, row) }); // ---- game_melee_attack -------------------------------------------------------------------- - // Combat stance (ATTACKSTART/STOP + the owner-only ranged CANCEL_AUTO_REPEAT). - wire_insert(db.game_melee_attack(), "game_melee_attack.insert", &view, move |v, row| { - melee_engaged(v, shard, row) + // Combat stance (ATTACKSTART/STOP + the owner-only ranged CANCEL_AUTO_REPEAT), anchored on + // the attacker's indexed cell. + wire_insert(db.game_melee_attack(), "game_melee_attack.insert", &view, |v, row| { + melee_engaged(v, row) }); - wire_delete(db.game_melee_attack(), "game_melee_attack.delete", &view, move |v, row| { - melee_disengaged(v, shard, row) + wire_delete(db.game_melee_attack(), "game_melee_attack.delete", &view, |v, row| { + melee_disengaged(v, row) }); // ---- recipient-keyed private event relays ----------------------------------------------- @@ -671,22 +793,19 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId // ---- game_aura ------------------------------------------------------------------------------ // The aura composite: array sync (visibility-gated), self-only duration/run-speed/armor, - // and the stealth HIDE/REVEAL transitions. The A_STEALTH count is computed HERE on the pump, - // where this connection's cache is exactly post-change; everything else runs in the job. + // and the stealth HIDE/REVEAL transitions. The pump keeps the aura index current and takes + // the A_STEALTH count from it, row by row in callback order: a Stealth cast's two inserts + // count 1 then 2 (both hide), its two deletes count 1 then 0 (the second reveals). + // Everything else runs in the job. { let view = view.clone(); let coord = coord.clone(); - db.game_aura().on_insert(move |ctx, row| { - let n = ctx - .db - .game_aura() - .iter() - .filter(|a| { - a.target_guid == row.target_guid - && a.eff_kind == super::subscriptions::A_STEALTH - }) - .count(); - guarded("game_aura.insert", || aura_applied(&view, &coord, shard, row, n)); + db.game_aura().on_insert(move |_ctx, row| { + guarded("game_aura.insert", || { + view.auras.upsert(shard, row); + let n = view.auras.stealth_count(row.target_guid); + aura_applied(&view, &coord, row, n) + }); }); } { @@ -695,49 +814,47 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_aura().on_update(move |_ctx, old, row| { let expires_changed = old.expires_at != row.expires_at; guarded("game_aura.update", || { - aura_updated(&view, &coord, shard, row, expires_changed) + view.auras.upsert(shard, row); + aura_updated(&view, &coord, row, expires_changed) }); }); } { let view = view.clone(); let coord = coord.clone(); - db.game_aura().on_delete(move |ctx, row| { - let n = ctx - .db - .game_aura() - .iter() - .filter(|a| { - a.target_guid == row.target_guid - && a.eff_kind == super::subscriptions::A_STEALTH - }) - .count(); - guarded("game_aura.delete", || aura_removed(&view, &coord, shard, row, n)); + db.game_aura().on_delete(move |_ctx, row| { + guarded("game_aura.delete", || { + view.auras.remove(shard, row); + let n = view.auras.stealth_count(row.target_guid); + aura_removed(&view, &coord, row, n) + }); }); } // ---- game_spell_cast_event / game_spell_impact_event -------------------------------------- - // Cast visuals (the full cast-lock contract, per viewer) + deferred projectile impact logs. - wire_insert(db.game_spell_cast_event(), "game_spell_cast_event.insert", &view, move |v, row| { - cast_event_appeared(v, shard, row) + // Cast visuals (the full cast-lock contract, per viewer) + deferred projectile impact logs, + // both anchored on the caster's cell. + wire_insert(db.game_spell_cast_event(), "game_spell_cast_event.insert", &view, |v, row| { + cast_event_appeared(v, row) }); - wire_insert(db.game_spell_impact_event(), "game_spell_impact_event.insert", &view, move |v, row| { - impact_appeared(v, shard, row) + wire_insert(db.game_spell_impact_event(), "game_spell_impact_event.insert", &view, |v, row| { + impact_appeared(v, row) }); // ---- game_emote_event / game_chat_event / game_channel_event ------------------------------ - // The social tier. Emotes broadcast; say/yell range-gate per viewer; channel lines - // membership-gate per viewer — each inside the job, via the shared bodies. + // The social tier. Emotes and say/yell are range-bound, so both are anchored on the sender's + // cell (say/yell keep their yard-range gate inside the job); channel lines membership-gate + // per viewer inside the job. { let coord = coord.clone(); wire_insert(db.game_emote_event(), "game_emote_event.insert", &view, move |v, row| { - emote_appeared(v, &coord, shard, row) + emote_appeared(v, &coord, row) }); } { let coord = coord.clone(); wire_insert(db.game_chat_event(), "game_chat_event.insert", &view, move |v, row| { - chat_appeared(v, &coord, shard, row) + chat_appeared(v, &coord, row) }); } { @@ -905,7 +1022,7 @@ fn entity_key(row: &WorldEntity) -> CellKey { /// A row entered the world (or this shard). Index it, then offer a CREATE to everyone who can see /// its cell. -fn entity_appeared(view: &WorldView, shard: ShardId, row: &WorldEntity) { +fn entity_appeared(view: &Arc, shard: ShardId, row: &WorldEntity) { let key = entity_key(row); view.spatial .upsert_entity(EntityLayer::WorldEntity, row.guid, key, shard); @@ -922,7 +1039,7 @@ fn entity_appeared(view: &WorldView, shard: ShardId, row: &WorldEntity) { /// * sessions that could see the OLD cell but not the new one get a DESTROY — the packet the SDK /// used to produce as an `on_delete` when a row left a box; /// * nobody else is touched. -fn entity_changed(view: &WorldView, shard: ShardId, old: &WorldEntity, new: &WorldEntity) { +fn entity_changed(view: &Arc, shard: ShardId, old: &WorldEntity, new: &WorldEntity) { let key = entity_key(new); let previous = view .spatial @@ -988,22 +1105,27 @@ fn encounter_equip_changed(view: &WorldView, shard: ShardId, row: &EncounterEqui } } -fn offer_create_job(view: &WorldView, shard: ShardId, session: SessionId, row: &Arc) { +fn offer_create_job( + view: &Arc, + shard: ShardId, + session: SessionId, + row: &Arc, +) { let Some(viewer) = view.viewer(session) else { return; }; let Some(coord) = view.shard(shard) else { return; }; - let row = row.clone(); + let (view, row) = (view.clone(), row.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { - super::subscriptions::offer_peer_create_for(&coord, &viewer, &row) + super::subscriptions::offer_peer_create_for(&coord, &view, &viewer, &row) }); } fn update_job( - view: &WorldView, + view: &Arc, shard: ShardId, session: SessionId, old: &Arc, @@ -1015,10 +1137,10 @@ fn update_job( let Some(coord) = view.shard(shard) else { return; }; - let (old, new) = (old.clone(), new.clone()); + let (view, old, new) = (view.clone(), old.clone(), new.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { - super::subscriptions::relay_entity_update(&coord, &viewer, &old, &new) + super::subscriptions::relay_entity_update(&coord, &view, &viewer, &old, &new) }); } @@ -1609,12 +1731,34 @@ fn corpse_vanished(view: &WorldView, shard: ShardId, row: &Corpse) { } } +// =============================================================================================== +// Cell-anchored event dispatch. The Module stamps the actor's cell on the event row (or the +// actor's own row is in the index), so the audience is the actor's box rather than the shard; +// each job's per-viewer gate stays the final filter. +// =============================================================================================== + +/// Cells a yard range can reach from an anchor cell: the range in cells, rounded up, plus one +/// because the listener's anchor is its client position while the gate reads its last durable +/// row, and the two can sit one cell apart. +fn cells_within(range_yd: f32) -> i32 { + (range_yd / GRID_CELL_SIZE).ceil() as i32 + 1 +} + +fn combat_audience(view: &WorldView, row: &CombatEvent) -> Vec> { + let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); + view.cell_audience( + Some(key), + BOX_HALF_SPAN, + &[row.attacker_guid, row.target_guid], + ) +} + /// A swing landed → the full 097 packet fork (`combat_event_outbound`: ranged GO + delayed damage /// log / melee ATTACKERSTATEUPDATE / killing-blow ATTACKSTOP) per viewer, gated on that viewer's /// `created` set inside the job. -fn combat_event_appeared(view: &WorldView, shard: ShardId, row: &CombatEvent) { +fn combat_event_appeared(view: &WorldView, row: &CombatEvent) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in combat_audience(view, &row) { let row = row.clone(); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1623,11 +1767,19 @@ fn combat_event_appeared(view: &WorldView, shard: ShardId, row: &CombatEvent) { } } +/// An engagement row names no cell; the attacker's indexed cell anchors it. +fn melee_audience(view: &WorldView, row: &MeleeAttack) -> Vec> { + let key = view + .spatial + .entity_cell(EntityLayer::WorldEntity, row.attacker_guid); + view.cell_audience(key, BOX_HALF_SPAN, &[row.attacker_guid, row.target_guid]) +} + /// A melee/ranged engagement began → SMSG_ATTACKSTART per viewer (`melee_engage_outbound` runs /// the ranged skip + created gate in the job). -fn melee_engaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { +fn melee_engaged(view: &WorldView, row: &MeleeAttack) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in melee_audience(view, &row) { let row = row.clone(); let created = viewer.created.clone(); let tx = viewer.tx.clone(); @@ -1639,9 +1791,9 @@ fn melee_engaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { /// An engagement row was removed → ATTACKSTOP / the owner-only CANCEL_AUTO_REPEAT /// (`melee_disengage_outbound`, keyed on each viewer's own guid). -fn melee_disengaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { +fn melee_disengaged(view: &WorldView, row: &MeleeAttack) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in melee_audience(view, &row) { let row = row.clone(); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -1809,11 +1961,21 @@ fn group_member_mirror_changed(view: &WorldView, coord: &Coordinator, row: &Grou }); } +/// An aura names its target and nothing else; the target's indexed cell anchors it. A target +/// with no row (a unit already gone) leaves the owner leg as the only recipient, and the owner +/// is the only viewer whose gate could still pass. +fn aura_audience(view: &WorldView, row: &Aura) -> Vec> { + let key = view + .spatial + .entity_cell(EntityLayer::WorldEntity, row.target_guid); + view.cell_audience(key, BOX_HALF_SPAN, &[row.target_guid]) +} + /// An aura appeared → per viewer: array sync + self-only packets + the stealth HIDE transition /// (`aura_insert_outbound`, all gates inside the job against that viewer's own state). -fn aura_applied(view: &Arc, coord: &Coordinator, shard: ShardId, row: &Aura, stealth_count: usize) { +fn aura_applied(view: &Arc, coord: &Coordinator, row: &Aura, stealth_count: usize) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in aura_audience(view, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1831,14 +1993,15 @@ fn aura_applied(view: &Arc, coord: &Coordinator, shard: ShardId, row: } /// An aura changed → per viewer: array sync + refresh-only duration + self-only packets. -fn aura_updated(view: &Arc, coord: &Coordinator, shard: ShardId, row: &Aura, expires_changed: bool) { +fn aura_updated(view: &Arc, coord: &Coordinator, row: &Aura, expires_changed: bool) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { - let (row, coord) = (row.clone(), coord.clone()); + for viewer in aura_audience(view, &row) { + let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { super::subscriptions::aura_update_outbound( &coord, + &view, &viewer.created, viewer.self_guid, &row, @@ -1850,9 +2013,9 @@ fn aura_updated(view: &Arc, coord: &Coordinator, shard: ShardId, row: /// An aura expired/was removed → per viewer: array sync + self-only packets + the stealth REVEAL /// transition. -fn aura_removed(view: &Arc, coord: &Coordinator, shard: ShardId, row: &Aura, stealth_count: usize) { +fn aura_removed(view: &Arc, coord: &Coordinator, row: &Aura, stealth_count: usize) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in aura_audience(view, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1869,11 +2032,20 @@ fn aura_removed(view: &Arc, coord: &Coordinator, shard: ShardId, row: } } +fn cast_audience(view: &WorldView, row: &SpellCastEvent) -> Vec> { + let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); + view.cell_audience( + Some(key), + BOX_HALF_SPAN, + &[row.caster_guid, row.target_guid], + ) +} + /// A cast event landed → the full cast-lock sequence per viewer (`cast_event_outbound`: the /// caster-private branches key on each viewer's own guid inside the job). -fn cast_event_appeared(view: &WorldView, shard: ShardId, row: &SpellCastEvent) { +fn cast_event_appeared(view: &WorldView, row: &SpellCastEvent) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in cast_audience(view, &row) { let row = row.clone(); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -1883,11 +2055,19 @@ fn cast_event_appeared(view: &WorldView, shard: ShardId, row: &SpellCastEvent) { } } -/// A projectile impact landed → the floating damage number per viewer (broadcast, like the -/// per-player leg). -fn impact_appeared(view: &WorldView, shard: ShardId, row: &SpellImpactEvent) { +fn impact_audience(view: &WorldView, row: &SpellImpactEvent) -> Vec> { + let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); + view.cell_audience( + Some(key), + BOX_HALF_SPAN, + &[row.caster_guid, row.target_guid], + ) +} + +/// A projectile impact landed → the floating damage number per viewer that can see the caster. +fn impact_appeared(view: &WorldView, row: &SpellImpactEvent) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in impact_audience(view, &row) { let row = row.clone(); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1896,11 +2076,20 @@ fn impact_appeared(view: &WorldView, shard: ShardId, row: &SpellImpactEvent) { } } -/// An emote landed → SMSG_TEXT_EMOTE + SMSG_EMOTE per viewer on the shard (broadcast, no range -/// gate — parity with the per-player leg). Name resolve runs in the job against `coord`'s cache. -fn emote_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &EmoteEvent) { +fn emote_audience(view: &WorldView, row: &EmoteEvent) -> Vec> { + let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); + view.cell_audience( + Some(key), + BOX_HALF_SPAN, + &[row.sender_guid, row.target_guid], + ) +} + +/// An emote landed → SMSG_TEXT_EMOTE + SMSG_EMOTE per viewer that can see the sender. Name +/// resolve runs in the job against `coord`'s cache. +fn emote_appeared(view: &WorldView, coord: &Coordinator, row: &EmoteEvent) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in emote_audience(view, &row) { let (row, coord) = (row.clone(), coord.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1909,12 +2098,22 @@ fn emote_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &E } } +/// A chat line names its speaker and nothing else; the speaker's indexed cell anchors it, widened +/// to the kind's yard range (a yell carries 300 yd, four cells past the box). +fn chat_audience(view: &WorldView, row: &ChatEvent) -> Vec> { + let key = view + .spatial + .entity_cell(EntityLayer::WorldEntity, row.sender_guid); + let span = cells_within(super::subscriptions::chat_range_yd(row.chat_type)); + view.cell_audience(key, span, &[row.sender_guid, row.target_guid]) +} + /// A say/yell line landed → SMSG_MESSAGECHAT per viewer, `chat_in_range`-gated inside the job /// (`chat_event_outbound`: speaker echo, SAY/YELL radii, map+instance fence, coordinator-cache /// endpoint lookups). -fn chat_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &ChatEvent) { +fn chat_appeared(view: &WorldView, coord: &Coordinator, row: &ChatEvent) { let row = Arc::new(row.clone()); - for viewer in viewers_on_shard(view, shard) { + for viewer in chat_audience(view, &row) { let (row, coord) = (row.clone(), coord.clone()); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -2019,6 +2218,9 @@ fn duel_winner_audience(viewer_guid: u64, initiator_guid: u64, challenged_guid: viewer_guid != 0 && viewer_guid != initiator_guid && viewer_guid != challenged_guid } +#[cfg(test)] +mod cell_audience_tests; + #[cfg(test)] mod relay_bench; @@ -3044,7 +3246,7 @@ fn enqueue(tx: &SessionTx, job: impl FnOnce() -> Vec + Send + 'static) /// The subscription engine used to produce these as `on_insert`/`on_delete` when a box was /// re-subscribed. Runs on the session's OWN movement thread — never on a pump — and does no I/O: /// the entering rows are read out of the shard coordinator caches already in memory. -pub(crate) fn recenter(view: &WorldView, viewer: &Arc, map_id: u32, x: f32, y: f32) { +pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, x: f32, y: f32) { let key = CellKey::of_position(map_id, viewer.instance_id, x, y); let Some(delta) = view.spatial.move_viewer_delta(viewer.session, key) else { return; // no crossing: the by-far-most-common per-heartbeat call, and it allocates nothing @@ -3085,9 +3287,9 @@ pub(crate) fn recenter(view: &WorldView, viewer: &Arc, map_id: u32, x: f else { continue; }; - let (viewer, row) = (viewer.clone(), Arc::new(row)); + let (view, viewer, row) = (view.clone(), viewer.clone(), Arc::new(row)); enqueue(&viewer.tx.clone(), move || { - super::subscriptions::offer_peer_create_for(&coord, &viewer, &row) + super::subscriptions::offer_peer_create_for(&coord, &view, &viewer, &row) }); } } @@ -3138,7 +3340,7 @@ pub(crate) fn sweep_into_view(view: &WorldView, viewer: &Arc) { else { continue; }; - for o in super::subscriptions::offer_peer_create_for(&coord, viewer, &row) { + for o in super::subscriptions::offer_peer_create_for(&coord, view, viewer, &row) { let _ = viewer.tx.send(o); } } @@ -3175,6 +3377,7 @@ pub(crate) fn sweep_into_view(view: &WorldView, viewer: &Arc) { /// entity happens to move. pub(crate) fn seed_from_caches(view: &WorldView) { let shards = view.shards.read().unwrap().clone(); + let mut auras = Vec::new(); for (id, coord) in shards.iter().enumerate() { let guard = coord.0.coord(); for row in guard.conn.db.game_world_entity().iter() { @@ -3189,12 +3392,15 @@ pub(crate) fn seed_from_caches(view: &WorldView) { id, ); } + auras.extend(guard.conn.db.game_aura().iter().map(|row| (id, row))); } + view.auras.replace_all(auras); let (entities, _, cells) = view.spatial.stats(EntityLayer::WorldEntity); let (gos, _, go_cells) = view.spatial.stats(EntityLayer::GameObject); + let (aura_rows, aura_units) = view.auras.stats(); log::info!( "shared AOI index seeded from {} shard cache(s): {entities} entities in {cells} cells, \ - {gos} gameobjects in {go_cells} cells", + {gos} gameobjects in {go_cells} cells, {aura_rows} auras on {aura_units} units", view.shard_count() ); } diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs new file mode 100644 index 000000000..3a373638b --- /dev/null +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -0,0 +1,686 @@ +//! The cell-anchored relays against the whole-shard-then-gate path they replaced. +//! +//! Every relay here used to enqueue one job per viewer on the shard and let the job's gate reject +//! the row. Now the actor's cell picks the candidates first. The property under test, over +//! randomised realms: filtering the candidates through the same gate yields exactly the set the +//! whole-shard path yielded, and the candidates are fewer than the shard. + +use super::*; +use crate::codec::property_tests::Rng; +use crate::stdb::bindings::{ + Aura, ChatEvent, CombatEvent, EmoteEvent, MeleeAttack, SpellCastEvent, SpellImpactEvent, +}; +use crate::stdb::subscriptions::{chat_in_range, chat_range_yd, A_STEALTH}; +use crate::world::{Outbound, SessionTx}; +use std::sync::mpsc::Receiver; + +const PLAYER_BASE: u64 = 0x1000; +const CREATURE_BASE: u64 = 0xF130_0000_0000_2000; +const YELL: u8 = 1; + +/// A randomised realm: players and creatures at random positions inside a `span_yd` square, one +/// partition, every viewer's `created` set holding exactly what its box shows it plus itself. +struct Realm { + view: Arc, + viewers: Vec>, + queues: Vec>, + positions: HashMap, + creatures: Vec, +} + +fn identity(guid: u64) -> spacetimedb_sdk::Identity { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&guid.to_le_bytes()); + spacetimedb_sdk::Identity::from_byte_array(bytes) +} + +fn viewer(session: SessionId, self_guid: u64, tx: SessionTx) -> Arc { + Arc::new(Viewer { + session, + self_guid, + bound_identity: identity(self_guid), + map_id: 0, + instance_id: 0, + zone_id: 0.into(), + tx, + created: Arc::new(Mutex::new(HashSet::from([self_guid]))), + gates: Arc::new(ViewerGates::default()), + skill_slots: Arc::new(Mutex::new((HashMap::new(), 0))), + explored: Mutex::new(ExplorationReplay::default()), + motion_pending: Arc::new(MotionPending::default()), + }) +} + +fn realm(rng: &mut Rng, players: usize, creatures: usize, span_yd: f32) -> Realm { + let view = Arc::new(WorldView::new(true)); + let position = |rng: &mut Rng| { + let x = rng.below(span_yd as usize) as f32; + let y = rng.below(span_yd as usize) as f32; + (x, y) + }; + let mut positions = HashMap::new(); + let mut viewers = Vec::new(); + let mut queues = Vec::new(); + for i in 0..players { + let guid = PLAYER_BASE + i as u64; + let (x, y) = position(rng); + let key = CellKey::of_position(0, 0, x, y); + view.spatial + .upsert_entity(EntityLayer::WorldEntity, guid, key, 0); + let (tx, rx) = SessionTx::with_depth(0); + let v = viewer(view.next_session_id(), guid, tx); + view.add_viewer_on_shard(v.clone(), key, 0); + positions.insert(guid, (x, y)); + viewers.push(v); + queues.push(rx); + } + let creatures: Vec = (0..creatures) + .map(|i| { + let guid = CREATURE_BASE + i as u64; + let (x, y) = position(rng); + view.spatial.upsert_entity( + EntityLayer::WorldEntity, + guid, + CellKey::of_position(0, 0, x, y), + 0, + ); + positions.insert(guid, (x, y)); + guid + }) + .collect(); + for v in &viewers { + let visible = view + .spatial + .visible_entities(EntityLayer::WorldEntity, v.session); + v.created + .lock() + .unwrap() + .extend(visible.into_iter().map(|(guid, _)| guid)); + } + Realm { + view, + viewers, + queues, + positions, + creatures, + } +} + +impl Realm { + /// A random unit: a player two times in three, else a creature. + fn actor(&self, rng: &mut Rng) -> u64 { + if rng.below(3) < 2 { + self.viewers[rng.below(self.viewers.len())].self_guid + } else { + self.creatures[rng.below(self.creatures.len())] + } + } + + fn grid(&self, guid: u64) -> (i32, i32) { + let (x, y) = self.positions[&guid]; + lyracore_shared::spatial::grid_cell(x, y) + } + + fn all(&self) -> HashSet { + self.viewers.iter().map(|v| v.session).collect() + } + + /// The whole-shard path: every viewer, filtered by `gate`. + fn shard_then_gate(&self, gate: impl Fn(&Viewer) -> bool) -> HashSet { + self.viewers + .iter() + .filter(|v| gate(v)) + .map(|v| v.session) + .collect() + } + + /// The sessions whose writer queue holds at least one job, drained. + fn jobs_landed(&self) -> HashSet { + let mut landed = HashSet::new(); + for (v, rx) in self.viewers.iter().zip(&self.queues) { + while let Ok(out) = rx.try_recv() { + assert!( + matches!(out, Outbound::Job(_)), + "the pump enqueues jobs only" + ); + landed.insert(v.session); + } + } + landed + } + + fn owner(&self, guid: u64) -> Option { + self.view.session_of_owner(guid) + } +} + +fn sessions(audience: Vec>) -> HashSet { + audience.into_iter().map(|v| v.session).collect() +} + +fn created_contains(v: &Viewer, guid: u64) -> bool { + v.created.lock().unwrap().contains(&guid) +} + +fn combat(realm: &Realm, attacker: u64, target: u64) -> CombatEvent { + let (grid_x, grid_y) = realm.grid(attacker); + CombatEvent { + id: 1, + attacker_guid: attacker, + target_guid: target, + damage: 12, + hit_info: 0, + killing_blow: false, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + blocked_amount: 0, + ranged_spell_id: 0, + ammo_display_id: 0, + spell_swing: false, + impact_delay_ms: 0, + map_id: 0, + instance_id: 0, + grid_x, + grid_y, + } +} + +fn melee(attacker: u64, target: u64) -> MeleeAttack { + MeleeAttack { + attacker_guid: attacker, + target_guid: target, + last_swing_ms: 0, + ranged_spell_id: 0, + last_offhand_swing_ms: 0, + rout_ends_ms: 0, + pursuit_ends_ms: 0, + leash_x: 0.0, + leash_y: 0.0, + } +} + +fn cast(realm: &Realm, caster: u64, target: u64) -> SpellCastEvent { + let (grid_x, grid_y) = realm.grid(caster); + SpellCastEvent { + id: 1, + caster_guid: caster, + spell_id: 133, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + target_guid: target, + cast_time_ms: 0, + is_completion: true, + damage: 40, + school: 2, + is_crit: false, + resisted: 0, + absorbed: 0, + is_interrupted: false, + cooldown_ms: 0, + delay_ms: 0, + healed: 0, + is_proc_log: false, + swing_hit_info: 0, + client_initiated: false, + map_id: 0, + instance_id: 0, + grid_x, + grid_y, + failure_reason: 0, + } +} + +fn impact(realm: &Realm, caster: u64, target: u64) -> SpellImpactEvent { + let (grid_x, grid_y) = realm.grid(caster); + SpellImpactEvent { + id: 1, + caster_guid: caster, + target_guid: target, + spell_id: 116, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + damage: 33, + school: 4, + is_crit: false, + resisted: 0, + absorbed: 0, + map_id: 0, + instance_id: 0, + grid_x, + grid_y, + } +} + +fn emote(realm: &Realm, sender: u64, target: u64) -> EmoteEvent { + let (grid_x, grid_y) = realm.grid(sender); + EmoteEvent { + id: 1, + sender_guid: sender, + text_emote: 101, + emote_anim: 3, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + target_guid: target, + map_id: 0, + instance_id: 0, + grid_x, + grid_y, + } +} + +fn chat(sender: u64, chat_type: u8) -> ChatEvent { + ChatEvent { + id: 1, + sender_guid: sender, + chat_type, + language: 7, + message: "hello".to_string(), + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + target_guid: 0, + } +} + +fn aura(id: u64, target: u64, eff_kind: u8) -> Aura { + Aura { + id, + target_guid: target, + caster_guid: target, + spell_id: 1784, + slot: 0, + level: 60, + flags: 0x1f, + applied_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + expires_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + effect_id: id, + eff_kind, + amount: 0, + eff_p0: 0, + eff_p0_kind: 0, + eff_p1: 0, + period_ms: 0, + amount_remaining: 0, + stacks: 1, + next_tick_micros: 0, + channel_target: 0, + enters_combat: false, + proc_flags: 0, + proc_chance: 0, + proc_ppm: 0.0, + proc_ex: 0, + proc_school_mask: 0, + proc_family_name: 0, + proc_family_flags: 0, + proc_charges: 0, + proc_icd_ms: 0, + proc_ready_micros: 0, + } +} + +/// 24 players and 40 creatures over a 600 yd square: twelve cells a side, so a box covers well +/// under half of it and the whole-shard path and the cell path differ on most events. +fn spread_realm(rng: &mut Rng) -> Realm { + realm(rng, 24, 40, 600.0) +} + +// =============================================================================================== +// Gated relays: the candidates filtered by the job's gate equal the shard filtered by the gate. +// =============================================================================================== + +/// A swing reaches exactly the viewers whose `created` set holds the attacker, and the pump +/// enqueues for the attacker's box and the two owners only. +#[test] +fn combat_candidates_gate_to_exactly_the_whole_shard_recipients() { + let mut rng = Rng::new(0xC0A7_0001); + let mut skipped_someone = false; + for _ in 0..200 { + let realm = spread_realm(&mut rng); + let attacker = realm.actor(&mut rng); + let target = realm.actor(&mut rng); + let row = combat(&realm, attacker, target); + let gate = |v: &Viewer| created_contains(v, attacker); + + let candidates = sessions(combat_audience(&realm.view, &row)); + combat_event_appeared(&realm.view, &row); + assert_eq!( + realm.jobs_landed(), + candidates, + "the dispatch enqueues for the audience" + ); + + let gated: HashSet = realm + .viewers + .iter() + .filter(|v| candidates.contains(&v.session) && gate(v)) + .map(|v| v.session) + .collect(); + assert_eq!(gated, realm.shard_then_gate(gate), "attacker {attacker:#x}"); + skipped_someone |= candidates.len() < realm.viewers.len(); + } + assert!( + skipped_someone, + "a spread realm must leave some viewers out of the box" + ); +} + +/// An engagement row has no cell of its own; the attacker's indexed cell stands in, and the +/// `created` gate still admits exactly the whole-shard recipients. +#[test] +fn melee_candidates_gate_to_exactly_the_whole_shard_recipients() { + let mut rng = Rng::new(0x3E1E_E002); + for _ in 0..200 { + let realm = spread_realm(&mut rng); + let attacker = realm.actor(&mut rng); + let row = melee(attacker, realm.actor(&mut rng)); + let gate = |v: &Viewer| created_contains(v, attacker); + + let candidates = sessions(melee_audience(&realm.view, &row)); + melee_engaged(&realm.view, &row); + assert_eq!(realm.jobs_landed(), candidates); + melee_disengaged(&realm.view, &row); + assert_eq!(realm.jobs_landed(), candidates); + + let gated: HashSet = realm + .viewers + .iter() + .filter(|v| candidates.contains(&v.session) && gate(v)) + .map(|v| v.session) + .collect(); + assert_eq!(gated, realm.shard_then_gate(gate)); + } +} + +/// An aura job passes its gate for the target itself, for a viewer showing the target, or (the +/// stealth reveal) for a viewer whose box covers the target. The target's cell admits all three. +#[test] +fn aura_candidates_gate_to_exactly_the_whole_shard_recipients() { + let mut rng = Rng::new(0xA07A_0003); + let mut skipped_someone = false; + for trial in 0..200 { + let realm = spread_realm(&mut rng); + let target = realm.actor(&mut rng); + let kind = if trial % 2 == 0 { A_STEALTH } else { 0xA0 }; + let row = aura(1, target, kind); + let gate = |v: &Viewer| { + v.self_guid == target + || created_contains(v, target) + || realm + .view + .spatial + .can_see(EntityLayer::WorldEntity, v.session, target) + }; + + let candidates = sessions(aura_audience(&realm.view, &row)); + let gated: HashSet = realm + .viewers + .iter() + .filter(|v| candidates.contains(&v.session) && gate(v)) + .map(|v| v.session) + .collect(); + assert_eq!(gated, realm.shard_then_gate(gate), "target {target:#x}"); + skipped_someone |= candidates.len() < realm.viewers.len(); + } + assert!(skipped_someone); +} + +/// A unit that is already gone from the index (its row deleted in the same transaction) can be +/// on nobody's screen, so only the owner leg remains. +#[test] +fn an_aura_on_an_unindexed_unit_reaches_its_owner_only() { + let mut rng = Rng::new(0xA07A_0004); + let realm = spread_realm(&mut rng); + let player = realm.viewers[3].self_guid; + realm + .view + .spatial + .remove_entity(EntityLayer::WorldEntity, player); + assert_eq!( + sessions(aura_audience(&realm.view, &aura(1, player, 0xA0))), + HashSet::from([realm.viewers[3].session]) + ); + let creature = realm.creatures[0]; + realm + .view + .spatial + .remove_entity(EntityLayer::WorldEntity, creature); + assert!(aura_audience(&realm.view, &aura(2, creature, 0xA0)).is_empty()); +} + +// =============================================================================================== +// Ungated relays: the audience is the actor's box plus the two named owners. +// =============================================================================================== + +/// The viewers showing the actor (their `created` sets were seeded from the index's forward +/// direction) plus the actor's and target's own sessions, whether or not they can see the cell. +fn box_plus_owners(realm: &Realm, actor: u64, target: u64) -> HashSet { + let mut want: HashSet = realm + .viewers + .iter() + .filter(|v| created_contains(v, actor)) + .map(|v| v.session) + .collect(); + want.extend(realm.owner(actor)); + want.extend(realm.owner(target)); + want +} + +#[test] +fn cast_impact_and_emote_reach_the_actors_box_and_both_owners() { + let mut rng = Rng::new(0xCA57_0005); + let mut skipped_someone = false; + for _ in 0..200 { + let realm = spread_realm(&mut rng); + let actor = realm.actor(&mut rng); + let target = realm.actor(&mut rng); + let want = box_plus_owners(&realm, actor, target); + + let row = cast(&realm, actor, target); + assert_eq!( + sessions(cast_audience(&realm.view, &row)), + want, + "cast by {actor:#x}" + ); + cast_event_appeared(&realm.view, &row); + assert_eq!(realm.jobs_landed(), want); + + let row = impact(&realm, actor, target); + assert_eq!( + sessions(impact_audience(&realm.view, &row)), + want, + "impact by {actor:#x}" + ); + impact_appeared(&realm.view, &row); + assert_eq!(realm.jobs_landed(), want); + + let row = emote(&realm, actor, target); + assert_eq!( + sessions(emote_audience(&realm.view, &row)), + want, + "emote by {actor:#x}" + ); + + skipped_someone |= want.len() < realm.viewers.len(); + } + assert!(skipped_someone); +} + +// =============================================================================================== +// Chat: the widened span never loses a listener the yard-range gate would admit. +// =============================================================================================== + +/// The job's gate, over the positions the fixture placed the units at. +fn hears(realm: &Realm, speaker: u64, listener: &Viewer, chat_type: u8) -> bool { + if listener.self_guid == speaker { + return true; + } + let (sx, sy) = realm.positions[&speaker]; + let (lx, ly) = realm.positions[&listener.self_guid]; + let range = chat_range_yd(chat_type); + chat_in_range(0, 0, sx, sy, 0, 0, lx, ly, range * range) +} + +#[test] +fn say_and_yell_candidates_gate_to_exactly_the_whole_shard_listeners() { + let mut rng = Rng::new(0x5A11_0006); + let mut yell_skipped_someone = false; + for trial in 0..300 { + // 1200 yd a side: a yell (300 yd) reaches a corner of it, never all of it. + let realm = realm(&mut rng, 30, 20, 1200.0); + let speaker = realm.actor(&mut rng); + let chat_type = if trial % 2 == 0 { YELL } else { 0 }; + let row = chat(speaker, chat_type); + let gate = |v: &Viewer| hears(&realm, speaker, v, chat_type); + + let candidates = sessions(chat_audience(&realm.view, &row)); + let gated: HashSet = realm + .viewers + .iter() + .filter(|v| candidates.contains(&v.session) && gate(v)) + .map(|v| v.session) + .collect(); + assert_eq!( + gated, + realm.shard_then_gate(gate), + "speaker {speaker:#x} kind {chat_type}" + ); + if chat_type == YELL { + yell_skipped_someone |= candidates.len() < realm.viewers.len(); + } else { + assert!( + candidates.len() < realm.all().len(), + "a say over a 1200 yd square never reaches everyone" + ); + } + } + assert!(yell_skipped_someone); +} + +/// A speaker with no indexed row cannot be placed, so nobody but the speaker hears it, which is +/// what the job's gate decides for the others too (no speaker row, no packet). +#[test] +fn chat_from_an_unindexed_speaker_reaches_the_speaker_only() { + let mut rng = Rng::new(0x5A11_0007); + let realm = spread_realm(&mut rng); + let speaker = realm.viewers[0].self_guid; + realm + .view + .spatial + .remove_entity(EntityLayer::WorldEntity, speaker); + assert_eq!( + sessions(chat_audience(&realm.view, &chat(speaker, YELL))), + HashSet::from([realm.viewers[0].session]) + ); +} + +// =============================================================================================== +// The aura index +// =============================================================================================== + +#[test] +fn aura_index_insert_update_delete_by_shard_and_id() { + let index = AuraIndex::default(); + index.upsert(0, &aura(1, 77, A_STEALTH)); + index.upsert(0, &aura(2, 77, 0xA0)); + index.upsert(1, &aura(1, 78, 0xA0)); // shard 1 reuses id 1 for another unit + assert_eq!(index.stealth_count(77), 1); + assert!(index.is_stealthed(77)); + assert!(!index.is_stealthed(78)); + assert_eq!(index.on_target(77).len(), 2); + assert_eq!(index.stats(), (3, 2)); + + // An update lands on the same (shard, id) and replaces the row. + let mut refreshed = aura(2, 77, 0xA0); + refreshed.amount = 25; + index.upsert(0, &refreshed); + assert_eq!(index.on_target(77).len(), 2); + assert_eq!( + index + .on_target(77) + .iter() + .find(|a| a.id == 2) + .map(|a| a.amount), + Some(25) + ); + + // A delete on shard 1 of id 1 must not touch shard 0's id 1. + index.remove(1, &aura(1, 78, 0xA0)); + assert!(index.on_target(78).is_empty()); + assert_eq!(index.stealth_count(77), 1); + index.remove(0, &aura(1, 77, A_STEALTH)); + assert_eq!(index.stealth_count(77), 0); + index.remove(0, &aura(1, 77, A_STEALTH)); // idempotent + assert_eq!(index.on_target(77).len(), 1); + index.remove(0, &aura(2, 77, 0xA0)); + assert_eq!(index.stats(), (0, 0)); +} + +#[test] +fn aura_index_reseed_drops_rows_the_caches_no_longer_hold() { + let index = AuraIndex::default(); + index.upsert(0, &aura(1, 77, 0xA0)); + index.upsert(0, &aura(2, 77, 0xA0)); + index.replace_all([(0, aura(2, 77, 0xA0)), (1, aura(9, 80, A_STEALTH))]); + assert_eq!(index.on_target(77).len(), 1); + assert!(index.is_stealthed(80)); + assert_eq!(index.stats(), (2, 2)); +} + +/// The aura audience follows the viewer's anchor: walking into the target's box adds the viewer, +/// walking out removes it, with nothing re-registered. +#[test] +fn aura_audience_follows_a_viewer_recenter() { + let view = WorldView::new(true); + let (tx, _rx) = SessionTx::with_depth(0); + let watcher = viewer(1, PLAYER_BASE, tx); + view.add_viewer_on_shard(watcher.clone(), CellKey::at(0, 0, 0, 0), 0); + let stealther = CREATURE_BASE; + view.spatial.upsert_entity( + EntityLayer::WorldEntity, + stealther, + CellKey::at(0, 0, 10, 10), + 0, + ); + let row = aura(1, stealther, A_STEALTH); + + assert!(aura_audience(&view, &row).is_empty()); + view.spatial + .move_viewer_delta(watcher.session, CellKey::at(0, 0, 9, 9)); + assert_eq!(sessions(aura_audience(&view, &row)), HashSet::from([1])); + view.spatial + .move_viewer_delta(watcher.session, CellKey::at(0, 0, 20, 20)); + assert!(aura_audience(&view, &row).is_empty()); + // The target moves under a resident viewer. + view.spatial.upsert_entity( + EntityLayer::WorldEntity, + stealther, + CellKey::at(0, 0, 21, 21), + 0, + ); + assert_eq!(sessions(aura_audience(&view, &row)), HashSet::from([1])); +} + +/// Every aura read the relays make goes through the index; a whole-cache scan must not creep +/// back into a job or onto the pump. +#[test] +fn aura_relays_never_scan_the_aura_cache() { + let subscriptions = include_str!("../subscriptions.rs"); + for signature in [ + "pub(crate) fn offer_peer_create_for", + "pub(crate) fn aura_insert_outbound", + "pub(crate) fn aura_update_outbound", + "pub(crate) fn aura_delete_outbound", + ] { + let body = crate::test_scan::code_of(subscriptions, signature); + assert!( + !body.contains("game_aura()"), + "{signature} scans the aura cache" + ); + assert!( + body.contains("view.auras."), + "{signature} must read the aura index" + ); + } + let arm = + crate::test_scan::code_of(include_str!("../world_view.rs"), "pub(crate) fn arm_shard"); + assert!( + !arm.contains(".game_aura().iter()"), + "the pump scans the aura cache" + ); + assert_eq!(arm.matches("view.auras.upsert(shard, row)").count(), 2); + assert_eq!(arm.matches("view.auras.remove(shard, row)").count(), 1); +} diff --git a/gateway/src/stdb/world_view/relay_bench.rs b/gateway/src/stdb/world_view/relay_bench.rs index da3291d72..70baaaeb2 100644 --- a/gateway/src/stdb/world_view/relay_bench.rs +++ b/gateway/src/stdb/world_view/relay_bench.rs @@ -2,12 +2,13 @@ //! //! Run with `cargo test -p lyracore-gateway --release relay_scaling -- --ignored --nocapture`. //! It prints one row per population size: the pump cost of a burst of combat, cast and impact -//! events, the writer-side cost of draining every job those enqueued, and the aura relay's -//! audience and array-sync cost. Ignored by default because it is a measurement, not a check. +//! events, the writer-side cost of draining every job those enqueued, the aura relay's audience +//! and array-sync cost, and the yell audience. Ignored by default because it is a measurement, +//! not a check. use super::*; use crate::codec::property_tests::Rng; -use crate::stdb::bindings::{Aura, CombatEvent, SpellCastEvent, SpellImpactEvent}; +use crate::stdb::bindings::{Aura, ChatEvent, CombatEvent, SpellCastEvent, SpellImpactEvent}; use crate::world::{Outbound, SessionTx}; use std::sync::mpsc::Receiver; use std::time::{Duration, Instant}; @@ -87,7 +88,14 @@ fn aura(id: u64, target_guid: u64, eff_kind: u8) -> Aura { /// player spread over every entity. fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { let side = ((players / PLAYERS_PER_CELL) as f64).sqrt().ceil() as i32; - let cell = |rng: &mut Rng| CellKey::at(0, 0, rng.below(side as usize) as i32, rng.below(side as usize) as i32); + let cell = |rng: &mut Rng| { + CellKey::at( + 0, + 0, + rng.below(side as usize) as i32, + rng.below(side as usize) as i32, + ) + }; let view = Arc::new(WorldView::new(true)); let mut viewers = Vec::with_capacity(players); let mut queues = Vec::with_capacity(players); @@ -127,10 +135,16 @@ fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { } else { creatures[pick - players].0 }; - let kind = if i % 20 == 0 { super::super::subscriptions::A_STEALTH } else { 0xA0 }; + let kind = if i % 20 == 0 { + super::super::subscriptions::A_STEALTH + } else { + 0xA0 + }; aura(1 + i as u64, target, kind) }) .collect(); + view.auras + .replace_all(auras.iter().map(|row| (0, row.clone()))); SyntheticRealm { view, viewers, @@ -223,6 +237,18 @@ impl SyntheticRealm { } } + fn yell_row(&self, speaker: usize) -> ChatEvent { + ChatEvent { + id: speaker as u64, + sender_guid: self.viewers[speaker].self_guid, + chat_type: 1, + language: 7, + message: "LFG".to_string(), + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + target_guid: 0, + } + } + /// Run every queued job on every writer queue: `(jobs run, packets produced)`. fn drain(&self) -> (usize, usize) { let (mut jobs, mut packets) = (0, 0); @@ -277,7 +303,7 @@ fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Durati for _ in 0..AURA_BURST { let row = &realm.auras[rng.below(realm.auras.len())]; let started = Instant::now(); - let recipients = realm.view.viewers_on_shard(0); + let recipients = aura_audience(&realm.view, row); audience += started.elapsed(); candidates += recipients.len(); for viewer in recipients { @@ -288,7 +314,7 @@ fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Durati } let started = Instant::now(); let _ = super::super::subscriptions::aura_sync( - realm.auras.iter().cloned(), + realm.view.auras.on_target(row.target_guid).into_iter(), row.target_guid, ); sync += started.elapsed(); @@ -312,16 +338,16 @@ fn relay_scaling() { let n = players; let combat = { let rows: Vec = (0..n).map(|i| realm.combat_row(i, &mut rng)).collect(); - burst(&realm, |i| combat_event_appeared(&realm.view, 0, &rows[i])) + burst(&realm, |i| combat_event_appeared(&realm.view, &rows[i])) }; let cast = { let rows: Vec = (0..n).map(|i| realm.cast_row(i, &mut rng)).collect(); - burst(&realm, |i| cast_event_appeared(&realm.view, 0, &rows[i])) + burst(&realm, |i| cast_event_appeared(&realm.view, &rows[i])) }; let impact = { let rows: Vec = (0..n).map(|i| realm.impact_row(i, &mut rng)).collect(); - burst(&realm, |i| impact_appeared(&realm.view, 0, &rows[i])) + burst(&realm, |i| impact_appeared(&realm.view, &rows[i])) }; for (name, b) in [("combat", combat), ("cast", cast), ("impact", impact)] { println!( @@ -344,7 +370,24 @@ fn relay_scaling() { micros_per(sync, AURA_BURST), synced as f64 / AURA_BURST as f64, ); + let yells: Vec = (0..n).map(|i| realm.yell_row(i)).collect(); + let started = Instant::now(); + let candidates: usize = yells + .iter() + .map(|row| chat_audience(&realm.view, row).len()) + .sum(); + let audience = started.elapsed(); + println!( + "| {players} | {} | {} | yell | {:.1} | {:.0} | n/a | n/a |", + realm.creatures.len(), + realm.auras.len(), + micros_per(audience, n), + candidates as f64 / n as f64, + ); } println!(); println!("aura columns: pump = audience selection, jobs = candidate viewers, drain = array syncs built, packets = viewers past the visibility gate"); + println!( + "yell columns: pump = audience selection over the widened span, jobs = candidate viewers" + ); } From cc0514801507f5e29a3d1943641e0ee315bfe5dd Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Fri, 4 Sep 2026 23:14:06 +0200 Subject: [PATCH 3/9] fix(gateway): preserve shard state across relay reconnects --- gateway/src/stdb/connection.rs | 188 +++- gateway/src/stdb/subscriptions.rs | 19 +- gateway/src/stdb/world_index.rs | 164 ++- gateway/src/stdb/world_view.rs | 941 ++++++++++++------ .../stdb/world_view/cell_audience_tests.rs | 434 +++++++- gateway/src/stdb/world_view/relay_bench.rs | 153 +-- 6 files changed, 1436 insertions(+), 463 deletions(-) diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index f5cddcad6..425456c7a 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -54,21 +54,70 @@ pub(crate) struct ShardSet { pub(crate) struct LiveConn { /// Privileged owner connection. Coordinators use its subscription cache for reads; call pipes /// use it only for the module's operator-gated reducer surface. - pub(crate) conn: DbConnection, + pub(crate) conn: Arc, /// Completes reducer waits as soon as this connection's transport dies. It belongs to the /// connection generation, so a watchdog replacement cannot fail calls on the fresh socket. pub(crate) reducer_completion: Arc, /// Keeps the SDK message-pump thread alive for the connection's lifetime. _pump: std::thread::JoinHandle<()>, + pump_work: tokio::sync::mpsc::UnboundedSender, /// Keeps this role's subscription active for the connection's lifetime. _sub: SubscriptionHandle, } +type PumpWork = Box; + +/// Run setup and SDK advancement on the same thread. SDK advancement is the external edge; +/// its cache update and row callbacks complete in one poll after the incoming message arrives. +async fn pump_messages>>( + conn: &C, + mut work: tokio::sync::mpsc::UnboundedReceiver>, + advance: impl Fn() -> F, +) -> std::result::Result<(), E> { + loop { + tokio::select! { + Some(work) = work.recv() => work(conn), + result = advance() => result?, + } + } +} + +/// Setup runs between complete SDK messages. The SDK consumes pending callback registrations +/// before its next incoming message, so registration and snapshot collection have no delta gap. +/// This relies on `spacetimedb-sdk` 2.7.1 `get_message` draining pending mutations first. +fn start_pump( + conn: &Arc, + role: &str, +) -> Result<( + std::thread::JoinHandle<()>, + tokio::sync::mpsc::UnboundedSender, +)> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + let conn = conn.clone(); + let role = role.to_owned(); + let thread = std::thread::Builder::new() + .name("stdb-pump".into()) + .spawn(move || { + let result = runtime.block_on(pump_messages(conn.as_ref(), rx, || { + conn.advance_one_message_async() + })); + if let Err(error) = result { + if !matches!(error, spacetimedb_sdk::Error::Disconnected) { + log::warn!("{role} pump stopped: {error}"); + } + } + })?; + Ok((thread, tx)) +} + impl LiveConn { - /// A connection generation is usable only while both its transport and subscription - /// are alive. A module republish can invalidate the latter without closing the socket. + /// A connection needs its transport, subscription, and pump. A Module republish can + /// invalidate the subscription while the socket remains connected. fn is_healthy(&self) -> bool { - self.conn.is_active() && self._sub.is_active() + self.conn.is_active() && self._sub.is_active() && !self._pump.is_finished() } } @@ -349,6 +398,20 @@ impl CoordinatorInner { }) } + /// Complete Relay setup on the current connection's pump, without retaining its slot lock. + pub(crate) fn on_pump(&self, work: impl FnOnce(&DbConnection) + Send + 'static) -> Result<()> { + let pump = self.coord().pump_work.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + pump.send(Box::new(move |conn| { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| work(conn))); + let _ = tx.send(outcome.map_err(|_| "Relay setup panicked")); + })) + .map_err(|_| anyhow!("{} Coordinator pump is stopped", self.db_name))?; + rx.recv_timeout(Duration::from_secs(15)) + .map_err(|error| anyhow!("{} Relay setup did not complete: {error}", self.db_name))? + .map_err(|error| anyhow!("{} {error}", self.db_name)) + } + /// The connection for the NEXT reducer call: round-robin over the call-pipe pool, /// skipping dead pipes; no pool (or every pipe dead) falls back to the watchdogged /// coordinator connection, which is exactly the behavior before the pool existed. @@ -447,7 +510,8 @@ fn connect_subscribed( .build() .map_err(|error| anyhow!("{role_label} build/connect failed: {error}"))?; - let pump = conn.run_threaded(); + let conn = Arc::new(conn); + let (pump, pump_work) = start_pump(&conn, &role_label)?; let (tx, rx) = std::sync::mpsc::channel::>(); let tx_err = tx.clone(); let sub = conn @@ -460,23 +524,32 @@ fn connect_subscribed( }) .subscribe(queries); - match rx.recv_timeout(Duration::from_secs(15)) { - Ok(Ok(())) => log::info!("{role_label} subscriptions applied"), - Ok(Err(error)) => return Err(anyhow!("{role_label} subscription error: {error}")), + let applied = match rx.recv_timeout(Duration::from_secs(15)) { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(anyhow!("{role_label} subscription error: {error}")), Err(_) => { let hint = subscription_timeout_hint .map(|hint| format!(" ({hint})")) .unwrap_or_default(); - return Err(anyhow!( + Err(anyhow!( "{role_label} subscriptions not applied within 15s{hint}" - )); + )) } + }; + if let Err(error) = applied { + let _ = conn.disconnect(); + if let Err(panic) = pump.join() { + log::warn!("{role_label} failed subscription pump panicked: {panic:?}"); + } + return Err(error); } + log::info!("{role_label} subscriptions applied"); Ok(LiveConn { conn, reducer_completion, _pump: pump, + pump_work, _sub: sub, }) } @@ -915,10 +988,10 @@ const COORDINATOR_WATCHDOG_POLL: Duration = Duration::from_secs(3); /// Background watchdog: detect a dropped coordinator connection and rebuild it IN PLACE so the gateway /// self-heals across a SpacetimeDB migration / network blip — no manual restart. Each poll it checks -/// BOTH liveness signals through [`LiveConn::is_healthy`]: the socket and the subscription. -/// Neither subsumes the other — a migration can invalidate the SUBSCRIPTION while the socket stays up +/// socket, subscription, and pump liveness through [`LiveConn::is_healthy`]. +/// A migration can invalidate the SUBSCRIPTION while the socket stays up /// (`conn.is_active()` stays true, and the SDK's subscription `on_disconnect` is a no-op), and a raw -/// socket drop leaves the subscription's status at `Applied` — so we heal when EITHER is down. On a drop +/// socket drop leaves the subscription's status at `Applied`. Recovery also replaces a stopped pump. On a drop /// it rebuilds a fresh connection + resubscribes OFF-LOCK (the up-to-15s apply must not block readers), /// swaps it in under the write lock (instant), then tears the OLD connection down off-lock (disconnect + /// join its pump thread, so it's reaped rather than detached-and-leaked — on the subscription-death path @@ -1417,6 +1490,74 @@ mod live_replacement_tests { } } +#[cfg(test)] +mod pump_tests { + use super::{pump_messages, PumpWork}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + use std::time::Duration; + use tokio::sync::mpsc; + + #[derive(Default)] + struct ExternalCache { + row: Mutex>, + projected: Mutex>, + registered: AtomicBool, + } + + #[tokio::test] + async fn a_quiet_pump_runs_setup_without_waiting_for_an_incoming_message() { + let (tx, rx) = mpsc::unbounded_channel::>(); + let cache = ExternalCache::default(); + let (done, finished) = tokio::sync::oneshot::channel(); + tx.send(Box::new(move |_| { + let _ = done.send(()); + })) + .unwrap(); + let pumping = pump_messages(&cache, rx, || std::future::pending::>()); + tokio::pin!(pumping); + tokio::select! { + _ = &mut pumping => panic!("the external connection is still open"), + outcome = tokio::time::timeout(Duration::from_secs(1), finished) => { + outcome.expect("setup must wake a quiet pump").unwrap(); + } + } + } + + #[tokio::test] + async fn a_delta_arriving_during_snapshot_setup_is_applied_after_the_snapshot() { + let cache = ExternalCache::default(); + *cache.row.lock().unwrap() = Some(1); + let (setup, work) = mpsc::unbounded_channel::>(); + let (incoming, messages) = mpsc::unbounded_channel(); + let messages = tokio::sync::Mutex::new(messages); + setup + .send(Box::new(move |cache| { + cache.registered.store(true, Ordering::Relaxed); + let snapshot = *cache.row.lock().unwrap(); + incoming.send(Some(2)).unwrap(); + incoming.send(None).unwrap(); + *cache.projected.lock().unwrap() = snapshot; + })) + .unwrap(); + + let result = pump_messages(&cache, work, || async { + let Some(Some(row)) = messages.lock().await.recv().await else { + return Err(()); + }; + *cache.row.lock().unwrap() = Some(row); + if cache.registered.load(Ordering::Relaxed) { + *cache.projected.lock().unwrap() = Some(row); + } + Ok(()) + }) + .await; + + assert_eq!(result, Err(())); + assert_eq!(*cache.projected.lock().unwrap(), Some(2)); + } +} + #[cfg(test)] mod recv_reducer_tests { use super::{ @@ -1860,7 +2001,7 @@ impl Coordinator { // after a coordinator reconnect (the watchdog treats a module republish as one), which is // what the `on_reconnect` hook below is for: the callbacks are bound to a `LiveConn` that // the swap replaces. - coordinator.arm_shared_world_view(); + coordinator.arm_shared_world_view()?; // The loot-roll relay: promotes each world shard's staging loot rolls onto realm-core and settles // resolved winners back down. A no-op loop on an unsharded gateway (`relay_tick` returns // immediately when `realm_store()` is `None`), so this costs a single-database deployment @@ -1887,19 +2028,22 @@ impl Coordinator { /// /// The shard ORDER fixes the `ShardId`s the index stores, so this runs once, before any session /// exists, and the vector is never reordered afterwards. The re-arm re-registers on the fresh - /// `LiveConn` — the index itself survives untouched, which is deliberate: a reconnect must not - /// blank every player's view. It does re-seed, because a fresh subscription's apply repopulates - /// the cache without firing per-row `on_insert`. - fn arm_shared_world_view(&self) { + /// `LiveConn`. Reconciliation replaces only that Shard's rows and keeps viewer registrations. + /// It removes missing rows because initial subscription apply does not replay their deletes. + fn arm_shared_world_view(&self) -> Result<()> { let view = self.world_view(); let shards = self.all_shards(); view.set_shards(shards.clone()); for (id, shard) in shards.iter().enumerate() { - super::world_view::arm_shard(view.clone(), shard.clone(), id); + super::world_view::arm_shard(view.clone(), shard.clone(), id)?; let (hook_view, hook_shard) = (view.clone(), shard.clone()); shard.0.on_reconnect.lock().unwrap().push(Arc::new(move || { - super::world_view::arm_shard(hook_view.clone(), hook_shard.clone(), id); - super::world_view::seed_from_caches(&hook_view); + if let Err(error) = + super::world_view::arm_shard(hook_view.clone(), hook_shard.clone(), id) + { + log::error!("{} Relay setup failed: {error:#}", hook_shard.shard_name()); + let _ = hook_shard.0.coord().conn.disconnect(); + } })); } // The cross-shard whisper/group twins (#22) ride realm-core's connection — armed only @@ -1919,7 +2063,7 @@ impl Coordinator { })); } } - super::world_view::seed_from_caches(&view); + Ok(()) } /// The database name this handle targets — the routing identity of every call made through it. diff --git a/gateway/src/stdb/subscriptions.rs b/gateway/src/stdb/subscriptions.rs index 35cf7a9c3..d6b065c0c 100644 --- a/gateway/src/stdb/subscriptions.rs +++ b/gateway/src/stdb/subscriptions.rs @@ -754,6 +754,7 @@ pub(crate) fn chat_range_yd(chat_type: u8) -> f32 { pub(crate) fn offer_peer_create_for( coord: &Coordinator, view: &WorldView, + shard: super::world_index::ShardId, viewer: &Viewer, row: &WorldEntity, ) -> Vec { @@ -766,7 +767,7 @@ pub(crate) fn offer_peer_create_for( if row.npc_flags & SPIRITHEALER_NPC_FLAG != 0 && !viewer_is_ghost { return Vec::new(); } - let row_is_stealthed = view.auras.is_stealthed(row.guid); + let row_is_stealthed = view.auras.is_stealthed(shard, row.guid); let guard = coord.0.coord(); let db = &guard.conn.db; if !peer_create_gate( @@ -873,6 +874,7 @@ fn append_encounter_equip_after_create( pub(crate) fn relay_entity_update( coord: &Coordinator, view: &WorldView, + shard: super::world_index::ShardId, viewer: &Viewer, old: &WorldEntity, new: &WorldEntity, @@ -882,7 +884,7 @@ pub(crate) fn relay_entity_update( if new.guid != viewer.self_guid { let shown = viewer.created.lock().unwrap().contains(&new.guid); if is_update_reentry(new.guid, viewer.self_guid, shown) { - return offer_peer_create_for(coord, view, viewer, new); + return offer_peer_create_for(coord, view, shard, viewer, new); } } let dynamic_flags = { @@ -918,7 +920,7 @@ pub(crate) fn relay_entity_update( drop(guard); for h in healers { if is_ghost { - out.extend(offer_peer_create_for(coord, view, viewer, &h)); + out.extend(offer_peer_create_for(coord, view, shard, viewer, &h)); } else if viewer.created.lock().unwrap().remove(&h.guid) { out.push(Outbound::One(ServerOpcodeMessage::SMSG_DESTROY_OBJECT( codec::build_destroy_object(h.guid), @@ -1632,6 +1634,7 @@ pub(crate) fn stealth_visibility( pub(crate) fn aura_insert_outbound( coord: &Coordinator, view: &WorldView, + shard: super::world_index::ShardId, session: u64, created: &Arc>>, self_guid: u64, @@ -1642,7 +1645,7 @@ pub(crate) fn aura_insert_outbound( // A stealth-hidden peer (not in `created`) must get NO per-peer relay: a partial VALUES on // a DESTROYed object is a client crash/desync vector. let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); - let current = visible.then(|| view.auras.on_target(row.target_guid)); + let current = visible.then(|| view.auras.on_target(shard, row.target_guid)); if let Some(current) = ¤t { out.push(aura_sync(current.iter().cloned(), row.target_guid)); } @@ -1677,6 +1680,7 @@ pub(crate) fn aura_insert_outbound( pub(crate) fn aura_update_outbound( coord: &Coordinator, view: &WorldView, + shard: super::world_index::ShardId, created: &Arc>>, self_guid: u64, row: &Aura, @@ -1684,7 +1688,7 @@ pub(crate) fn aura_update_outbound( ) -> Vec { let mut out = Vec::new(); let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); - let current = visible.then(|| view.auras.on_target(row.target_guid)); + let current = visible.then(|| view.auras.on_target(shard, row.target_guid)); if let Some(current) = ¤t { out.push(aura_sync(current.iter().cloned(), row.target_guid)); } @@ -1714,6 +1718,7 @@ pub(crate) fn aura_update_outbound( pub(crate) fn aura_delete_outbound( coord: &Coordinator, view: &WorldView, + shard: super::world_index::ShardId, session: u64, created: &Arc>>, self_guid: u64, @@ -1723,7 +1728,7 @@ pub(crate) fn aura_delete_outbound( let mut out = Vec::new(); let visible = row.target_guid == self_guid || created.lock().unwrap().contains(&row.target_guid); if visible { - let current = view.auras.on_target(row.target_guid); + let current = view.auras.on_target(shard, row.target_guid); out.push(aura_sync(current.iter().cloned(), row.target_guid)); if let Some(o) = run_speed_packet(current.into_iter(), row, self_guid) { out.push(o); @@ -6199,7 +6204,7 @@ mod tests { /// freeze after their first step, or never move at all. Same for the creature-leg twin. #[test] fn both_halves_of_the_motion_and_spline_relays_are_registered_286() { - let arm = decommented(top_level_fn_body_of("world_view.rs", "arm_shard")); + let arm = decommented(top_level_fn_body_of("world_view.rs", "register_shard_callbacks")); // #490 factored every registration through `wire_insert`/`wire_update` (world_view.rs), // so the literal `.on_insert(`/`.on_update(` chain off the table handle is gone from // `arm_shard`'s own body — what's left to scan for is the (helper, table, label) triple diff --git a/gateway/src/stdb/world_index.rs b/gateway/src/stdb/world_index.rs index f3c77973f..fafbbbf7b 100644 --- a/gateway/src/stdb/world_index.rs +++ b/gateway/src/stdb/world_index.rs @@ -164,9 +164,9 @@ pub struct RecenterDelta { struct EntityIndex { /// cell → the guids resident in it. The forward direction of the interest question. cell_entities: HashMap>, - /// guid → where it is and whose cache holds it. Also the "did it move?" memory that makes - /// [`WorldIndex::upsert_entity`] O(1) instead of a search. - entity_cell: HashMap, + /// GUID to cell, Shard, and pet owner. The owner survives a reconnect so a missing + /// pet clears its Character's action bar as well as its client object. + entity_cell: HashMap, } #[derive(Default)] @@ -239,10 +239,14 @@ impl WorldIndex { guid: u64, key: CellKey, shard: ShardId, + owner_guid: u64, ) -> Option { let mut inner = self.lock(); let entities = Self::layer_mut(&mut inner, layer); - let previous = entities.entity_cell.insert(guid, (key, shard)).map(|(k, _)| k); + let previous = entities + .entity_cell + .insert(guid, (key, shard, owner_guid)) + .map(|(k, _, _)| k); match previous { Some(old) if old == key => return Some(old), Some(old) => { @@ -259,29 +263,36 @@ impl WorldIndex { previous } - /// (Inspection only — dispatch uses the shard carried by visibility results.) - #[cfg(test)] /// Which shard's coordinator cache holds this entity's row. pub fn shard_of(&self, layer: EntityLayer, guid: u64) -> Option { let inner = self.lock(); Self::layer(&inner, layer) .entity_cell .get(&guid) - .map(|(_, s)| *s) + .map(|(_, s, _)| *s) } - /// Forget an entity (its row was deleted). Returns the cell it was in, if it was known. - pub fn remove_entity(&self, layer: EntityLayer, guid: u64) -> Option { + /// Remove a row only from its current Shard. Returns its old cell and pet owner GUID. + pub fn remove_entity( + &self, + layer: EntityLayer, + guid: u64, + shard: ShardId, + ) -> Option<(CellKey, u64)> { let mut inner = self.lock(); let entities = Self::layer_mut(&mut inner, layer); - let (key, _) = entities.entity_cell.remove(&guid)?; + let &(key, owner, owner_guid) = entities.entity_cell.get(&guid)?; + if owner != shard { + return None; + } + entities.entity_cell.remove(&guid); if let Some(set) = entities.cell_entities.get_mut(&key) { set.remove(&guid); if set.is_empty() { entities.cell_entities.remove(&key); } } - Some(key) + Some((key, owner_guid)) } /// Where the index believes `guid` is: the anchor of a relay driven by a table with no cell of @@ -289,7 +300,71 @@ impl WorldIndex { /// seen (a row on a table it does not index, or one deleted already). pub fn entity_cell(&self, layer: EntityLayer, guid: u64) -> Option { let inner = self.lock(); - Self::layer(&inner, layer).entity_cell.get(&guid).map(|(k, _)| *k) + Self::layer(&inner, layer) + .entity_cell + .get(&guid) + .map(|(k, _, _)| *k) + } + + pub fn entity_cell_on_shard( + &self, + layer: EntityLayer, + guid: u64, + shard: ShardId, + ) -> Option { + let inner = self.lock(); + Self::layer(&inner, layer) + .entity_cell + .get(&guid) + .and_then(|(key, owner, _)| (*owner == shard).then_some(*key)) + } + + /// Replace one shard's snapshot, preserving viewers and rows already owned by another shard. + /// Rows and returned removals carry GUID, cell, and pet owner GUID. Former viewers can + /// drop vanished or moved rows and clear a vanished pet's action bar. + pub fn replace_shard( + &self, + layer: EntityLayer, + shard: ShardId, + rows: impl IntoIterator, + ) -> Vec<(u64, CellKey, u64)> { + let mut inner = self.lock(); + let entities = Self::layer_mut(&mut inner, layer); + let fresh: HashMap<_, _> = rows + .into_iter() + .filter(|(guid, _, _)| { + entities + .entity_cell + .get(guid) + .is_none_or(|(_, owner, _)| *owner == shard) + }) + .map(|(guid, key, owner)| (guid, (key, owner))) + .collect(); + let old: Vec<_> = entities + .entity_cell + .iter() + .filter(|(_, (_, owner, _))| *owner == shard) + .map(|(guid, (key, _, owner_guid))| (*guid, *key, *owner_guid)) + .collect(); + let mut removed = Vec::new(); + for (guid, key, owner_guid) in old { + if fresh.get(&guid).map(|(cell, _)| cell) == Some(&key) { + continue; + } + entities.entity_cell.remove(&guid); + if let Some(guids) = entities.cell_entities.get_mut(&key) { + guids.remove(&guid); + if guids.is_empty() { + entities.cell_entities.remove(&key); + } + } + removed.push((guid, key, owner_guid)); + } + for (guid, (key, owner_guid)) in fresh { + entities.entity_cell.insert(guid, (key, shard, owner_guid)); + entities.cell_entities.entry(key).or_default().insert(guid); + } + removed } // --------------------------------------------------------------------------------------- @@ -403,7 +478,7 @@ impl WorldIndex { for cell in cells.difference(other) { if let Some(guids) = entities.cell_entities.get(cell) { for guid in guids { - let shard = entities.entity_cell.get(guid).map(|(_, s)| *s).unwrap_or(0); + let shard = entities.entity_cell.get(guid).map(|(_, s, _)| *s).unwrap_or(0); out.push((*guid, shard)); } } @@ -438,15 +513,15 @@ impl WorldIndex { return entities .entity_cell .iter() - .filter(|(_, (k, _))| k.partition() == partition) - .map(|(guid, (_, shard))| (*guid, *shard)) + .filter(|(_, (k, _, _))| k.partition() == partition) + .map(|(guid, (_, shard, _))| (*guid, *shard)) .collect(); } let mut out = Vec::new(); for cell in anchor.neighbourhood() { if let Some(guids) = entities.cell_entities.get(&cell) { for guid in guids { - if let Some((_, shard)) = entities.entity_cell.get(guid) { + if let Some((_, shard, _)) = entities.entity_cell.get(guid) { out.push((*guid, *shard)); } } @@ -465,7 +540,7 @@ impl WorldIndex { let Some(anchor) = inner.viewer_cell.get(&session) else { return false; }; - let Some((cell, _)) = Self::layer(&inner, layer).entity_cell.get(&guid) else { + let Some((cell, _, _)) = Self::layer(&inner, layer).entity_cell.get(&guid) else { return false; }; if anchor.partition() != cell.partition() { @@ -582,6 +657,7 @@ mod tests { r.guid, CellKey::at(r.map_id, r.instance_id, r.gx, r.gy), 0, + 0, ); } index @@ -730,6 +806,7 @@ mod tests { row.guid, CellKey::at(row.map_id, row.instance_id, row.gx, row.gy), 0, + 0, ); } } @@ -828,18 +905,47 @@ mod tests { let index = WorldIndex::new(true); let a = CellKey::at(0, 0, 0, 0); let b = CellKey::at(0, 0, 40, 40); // far outside a's box - index.upsert_entity(EntityLayer::WorldEntity, 5, a, 0); + index.upsert_entity(EntityLayer::WorldEntity, 5, a, 0, 0); index.add_viewer(1, a); assert_eq!(index_visible(&index, EntityLayer::WorldEntity, 1), HashSet::from([5])); - assert_eq!(index.upsert_entity(EntityLayer::WorldEntity, 5, b, 0), Some(a)); + assert_eq!(index.upsert_entity(EntityLayer::WorldEntity, 5, b, 0, 0), Some(a)); assert!( index_visible(&index, EntityLayer::WorldEntity, 1).is_empty(), "stale cell membership" ); assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 5), Some(b)); - assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5), Some(b)); + assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5, 0), Some((b, 0))); assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 5), None); - assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5), None, "removal is idempotent"); + assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5, 0), None, "removal is idempotent"); + } + + #[test] + fn a_shard_snapshot_replaces_its_rows_without_erasing_viewers_or_a_transfer_destination() { + let index = WorldIndex::new(true); + let near = CellKey::at(0, 0, 0, 0); + let far = CellKey::at(0, 0, 20, 20); + index.add_viewer(1, near); + index.upsert_entity(EntityLayer::WorldEntity, 10, near, 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 20, near, 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 30, near, 1, 0); + + let removed = index.replace_shard( + EntityLayer::WorldEntity, + 0, + [(20, far, 0), (30, far, 0), (40, near, 0)], + ); + assert_eq!( + removed.into_iter().collect::>(), + HashSet::from([(10, near, 0), (20, near, 0)]) + ); + assert_eq!(index.viewer_cell(1), Some(near)); + assert_eq!(index.shard_of(EntityLayer::WorldEntity, 30), Some(1)); + assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 30), Some(near)); + assert_eq!( + index_visible(&index, EntityLayer::WorldEntity, 1), + HashSet::from([30, 40]) + ); + assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 20), Some(far)); } #[test] @@ -852,7 +958,7 @@ mod tests { assert_eq!(index.viewer_cell(1), Some(CellKey::at(0, 0, 1, 0))); // Only one anchor is ever held: walking away must not leave the session visible from the // cell it started in. - index.upsert_entity(EntityLayer::WorldEntity, 5, CellKey::at(0, 0, 0, 0), 0); + index.upsert_entity(EntityLayer::WorldEntity, 5, CellKey::at(0, 0, 0, 0), 0, 0); index.move_viewer_delta(1, CellKey::at(0, 0, 30, 30)); assert!(index .viewers_of(EntityLayer::WorldEntity, CellKey::at(0, 0, 0, 0)) @@ -871,10 +977,10 @@ mod tests { #[test] fn the_aoi_off_hatch_sees_the_whole_partition_and_nothing_outside_it() { let index = WorldIndex::new(false); - index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0); - index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 500, 500), 0); - index.upsert_entity(EntityLayer::WorldEntity, 3, CellKey::at(0, 77, 0, 0), 0); // another instance - index.upsert_entity(EntityLayer::WorldEntity, 4, CellKey::at(1, 0, 0, 0), 0); // another map + index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 500, 500), 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 3, CellKey::at(0, 77, 0, 0), 0, 0); // another instance + index.upsert_entity(EntityLayer::WorldEntity, 4, CellKey::at(1, 0, 0, 0), 0, 0); // another map index.add_viewer(1, CellKey::at(0, 0, 0, 0)); assert_eq!(index_visible(&index, EntityLayer::WorldEntity, 1), HashSet::from([1, 2])); assert_eq!( @@ -931,9 +1037,9 @@ mod tests { #[test] fn stats_report_what_the_ops_line_prints() { let index = WorldIndex::new(true); - index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0); - index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 0, 0), 0); - index.upsert_entity(EntityLayer::WorldEntity, 3, CellKey::at(0, 0, 1, 0), 0); + index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 0, 0), 0, 0); + index.upsert_entity(EntityLayer::WorldEntity, 3, CellKey::at(0, 0, 1, 0), 0, 0); index.add_viewer(1, CellKey::at(0, 0, 0, 0)); assert_eq!(index.stats(EntityLayer::WorldEntity), (3, 1, 2)); } diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index 07400fd44..1cdb2e656 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -199,85 +199,84 @@ impl ViewerRegistry { } } -/// The aura cache indexed by target: every `game_aura` row, keyed by the unit it sits on, fed by -/// the same per-shard callbacks that relay it. Every aura read a relay makes is per target (the -/// array sync, the run-speed fold, the stealth count), and the SDK cache indexes rows by id only, -/// so each read used to scan the whole shard's auras. -/// -/// Rows are identified by `(shard, id)`: `id` is a per-shard autoincrement, so a transfer's late -/// cascade delete on the source shard must not evict the destination shard's row of the same id. +/// Auras indexed by shard and target. Transfer copies on different shards remain independent. #[derive(Default)] pub(crate) struct AuraIndex { - by_target: Mutex>>, + by_target: Mutex>>, } impl AuraIndex { - fn lock(&self) -> std::sync::MutexGuard<'_, HashMap>> { - self.by_target.lock().unwrap_or_else(|p| { - log::error!( - "aura index lock poisoned (a prior panic in a critical section) — recovering" - ); - p.into_inner() - }) + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<(ShardId, u64), Vec>> { + self.by_target.lock().unwrap_or_else(|p| p.into_inner()) } - /// Record or replace a row. A reconnect's initial apply replays resident rows as inserts, and - /// those land on their own `(shard, id)` instead of doubling. pub(crate) fn upsert(&self, shard: ShardId, row: &Aura) { let mut by_target = self.lock(); - let rows = by_target.entry(row.target_guid).or_default(); - match rows.iter_mut().find(|(s, a)| *s == shard && a.id == row.id) { - Some(slot) => slot.1 = row.clone(), - None => rows.push((shard, row.clone())), + let rows = by_target.entry((shard, row.target_guid)).or_default(); + match rows.iter_mut().find(|a| a.id == row.id) { + Some(slot) => *slot = row.clone(), + None => rows.push(row.clone()), } } - /// Forget a row. Idempotent. pub(crate) fn remove(&self, shard: ShardId, row: &Aura) { let mut by_target = self.lock(); - if let Some(rows) = by_target.get_mut(&row.target_guid) { - rows.retain(|(s, a)| !(*s == shard && a.id == row.id)); + let key = (shard, row.target_guid); + if let Some(rows) = by_target.get_mut(&key) { + rows.retain(|a| a.id != row.id); if rows.is_empty() { - by_target.remove(&row.target_guid); + by_target.remove(&key); } } } - /// Rebuild from the shard caches. The reconnect path: a fresh subscription's apply repopulates - /// the cache without firing a delete for a row that expired while the connection was down. - pub(crate) fn replace_all(&self, rows: impl IntoIterator) { - let mut fresh: HashMap> = HashMap::new(); - for (shard, row) in rows { - fresh.entry(row.target_guid).or_default().push((shard, row)); + /// Called on the shard's pump between messages, after its callbacks are registered. + pub(crate) fn replace_shard( + &self, + shard: ShardId, + rows: impl IntoIterator, + ) -> Vec { + let mut by_target = self.lock(); + let targets: Vec<_> = by_target + .keys() + .filter(|(owner, _)| *owner == shard) + .copied() + .collect(); + let old = targets + .into_iter() + .flat_map(|target| by_target.remove(&target).unwrap_or_default()) + .collect(); + for row in rows { + by_target + .entry((shard, row.target_guid)) + .or_default() + .push(row); } - *self.lock() = fresh; + old } - /// The auras on one unit, cloned out so no encoding ever runs under the lock. - pub(crate) fn on_target(&self, target_guid: u64) -> Vec { + pub(crate) fn on_target(&self, shard: ShardId, target_guid: u64) -> Vec { self.lock() - .get(&target_guid) - .map(|rows| rows.iter().map(|(_, a)| a.clone()).collect()) + .get(&(shard, target_guid)) + .cloned() .unwrap_or_default() } - /// How many `A_STEALTH` auras the unit carries right now. - pub(crate) fn stealth_count(&self, target_guid: u64) -> usize { + pub(crate) fn stealth_count(&self, shard: ShardId, target_guid: u64) -> usize { self.lock() - .get(&target_guid) + .get(&(shard, target_guid)) .map(|rows| { rows.iter() - .filter(|(_, a)| a.eff_kind == super::subscriptions::A_STEALTH) + .filter(|a| a.eff_kind == super::subscriptions::A_STEALTH) .count() }) .unwrap_or(0) } - pub(crate) fn is_stealthed(&self, target_guid: u64) -> bool { - self.stealth_count(target_guid) > 0 + pub(crate) fn is_stealthed(&self, shard: ShardId, target_guid: u64) -> bool { + self.stealth_count(shard, target_guid) > 0 } - /// Live sizes, for the seed log line: `(auras, units carrying one)`. pub(crate) fn stats(&self) -> (usize, usize) { let by_target = self.lock(); (by_target.values().map(Vec::len).sum(), by_target.len()) @@ -295,6 +294,7 @@ pub(crate) struct WorldView { /// dispatch; the order fixes the ids, so it must never be reordered after arming. shards: RwLock>, next_session: AtomicU64, + generations: Mutex>>>, } impl WorldView { @@ -305,6 +305,7 @@ impl WorldView { viewers: RwLock::new(ViewerRegistry::default()), shards: RwLock::new(Vec::new()), next_session: AtomicU64::new(1), + generations: Mutex::new(HashMap::new()), } } @@ -426,13 +427,20 @@ impl WorldView { /// Geometric recipients plus the row owner's session. The owner lookup lives in the viewer /// registry because it is session identity, not spatial state. - fn world_entity_recipients(&self, guid: u64, key: CellKey) -> Vec { + fn world_entity_recipients(&self, shard: ShardId, guid: u64, key: CellKey) -> Vec { let mut recipients = self.spatial.viewers_of(EntityLayer::WorldEntity, key); if let Some(owner) = self.session_of_owner(guid) { if !recipients.contains(&owner) { recipients.push(owner); } } + let registry = self.viewers.read().unwrap(); + recipients.retain(|session| { + registry + .by_session + .get(session) + .is_some_and(|v| v.shard == shard) + }); recipients } @@ -440,13 +448,14 @@ impl WorldView { /// of `key`, plus the sessions of the characters the row names. The owner leg keeps a /// self-addressed packet (the attacker's ATTACKSTOP, the caster's cast result, a player's own /// aura timer) reaching its owner while the owner's anchor and the row's cell disagree by a - /// heartbeat, and it is the whole audience when the anchor is unknown (`None`: the actor's - /// row is gone, so no other viewer has it on screen). + /// heartbeat. When the source Shard has no anchor, only its owners receive the event. + /// Every recipient must still belong to that Shard after a transfer. /// /// The per-viewer gate the job runs is the final filter, unchanged; this only stops the pump /// enqueueing a job for every viewer on the shard so it can reject the row later. fn cell_audience( &self, + shard: ShardId, key: Option, half_span: i32, owners: &[u64], @@ -465,6 +474,7 @@ impl WorldView { sessions .into_iter() .filter_map(|session| registry.by_session.get(&session)) + .filter(|registered| registered.shard == shard) .map(|registered| registered.viewer.clone()) .collect() } @@ -478,21 +488,64 @@ impl WorldView { *self.shards.write().unwrap() = shards; } - /// The number of shards registered — used by the ops log line and by the arming loop. - pub(crate) fn shard_count(&self) -> usize { - self.shards.read().unwrap().len() + fn with_new_generation(&self, shard: ShardId, body: impl FnOnce(ShardGeneration)) { + let current = self + .generations + .lock() + .unwrap() + .entry(shard) + .or_default() + .clone(); + let mut number = current.lock().unwrap_or_else(|p| p.into_inner()); + *number += 1; + body(ShardGeneration { + current: current.clone(), + number: *number, + }); + } + +} + +#[derive(Clone)] +struct ShardGeneration { + current: Arc>, + number: u64, +} + +impl ShardGeneration { + /// The lock also waits for an old callback already in progress before snapshot replacement. + fn run(generation: Option<&Self>, body: impl FnOnce()) { + if let Some(generation) = generation { + let current = generation.current.lock().unwrap_or_else(|p| p.into_inner()); + if *current != generation.number { + return; + } + body(); + } else { + body(); + } } } -/// Register the shared AOI relays on ONE shard's coordinator connection. -/// -/// Re-reads `coord.0.coord()` on every call, which is what makes it correct as the watchdog's -/// post-reconnect re-arm hook too: called again, it registers on the FRESH `LiveConn`. The old -/// connection's callbacks die with it, so nothing needs removing. -pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId) { - let guard = coord.0.coord(); - let db = &guard.conn.db; +/// Register Relays and reconcile the cache snapshot between messages on this Shard's pump. +/// The generation lock finishes any old callback first and rejects its later callbacks. +pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId) -> Result<()> { + let owner = coord.clone(); + coord.0.on_pump(move |conn| { + view.with_new_generation(shard, |generation| { + register_shard_callbacks(view.clone(), owner, shard, &conn.db, generation); + seed_shard_from_cache(&view, shard, &conn.db); + }); + }) +} +fn register_shard_callbacks( + view: Arc, + coord: Coordinator, + shard: ShardId, + db: &RemoteTables, + generation: ShardGeneration, +) { // ---- owner-scoped events --------------------------------------------------------------- // These callbacks address one viewer directly and survive coordinator reconnects because // `arm_shard` is re-run on the replacement connection. Every table here holds SHARD rows, @@ -505,22 +558,36 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_teleport_event(), "game_teleport_event.insert", &view, + Some(&generation), move |v, row| teleport_appeared(v, shard, row), ); - wire_insert_live(db.game_addon_message(), "game_addon_message.insert", &view, |v, row| { - addon_message_appeared(v, row) - }); - wire_insert_live(db.game_xp_event(), "game_xp_event.insert", &view, |v, row| { - xp_appeared(v, row) - }); - wire_insert_live(db.game_levelup_event(), "game_levelup_event.insert", &view, |v, row| { - levelup_appeared(v, row) - }); + wire_insert_live( + db.game_addon_message(), + "game_addon_message.insert", + &view, + Some(&generation), + |v, row| addon_message_appeared(v, row), + ); + wire_insert_live( + db.game_xp_event(), + "game_xp_event.insert", + &view, + Some(&generation), + |v, row| xp_appeared(v, row), + ); + wire_insert_live( + db.game_levelup_event(), + "game_levelup_event.insert", + &view, + Some(&generation), + |v, row| levelup_appeared(v, row), + ); let quest_insert_coord = coord.clone(); wire_insert( db.game_character_quest(), "game_character_quest.insert", &view, + Some(&generation), move |v, row| quest_inserted(v, &quest_insert_coord, shard, row), ); let quest_update_coord = coord.clone(); @@ -528,6 +595,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_character_quest(), "game_character_quest.update", &view, + Some(&generation), move |v, old, row| quest_updated(v, &quest_update_coord, shard, old, row), ); let quest_delete_coord = coord.clone(); @@ -535,6 +603,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_character_quest(), "game_character_quest.delete", &view, + Some(&generation), move |v, row| quest_deleted(v, &quest_delete_coord, shard, row), ); { @@ -543,6 +612,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_character_explored(), "game_character_explored.insert", &view, + Some(&generation), move |v, row| exploration_appeared(v, &coord, shard, row), ); } @@ -550,24 +620,28 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_player_reputation(), "game_player_reputation.insert", &view, + Some(&generation), move |v, row| reputation_appeared(v, shard, row), ); wire_update( db.game_player_reputation(), "game_player_reputation.update", &view, + Some(&generation), move |v, _old, row| reputation_appeared(v, shard, row), ); wire_insert( db.game_hunter_pet_protocol(), "game_hunter_pet_protocol.insert", &view, + Some(&generation), move |v, row| hunter_pet_appeared(v, shard, row), ); wire_update( db.game_hunter_pet_protocol(), "game_hunter_pet_protocol.update", &view, + Some(&generation), move |v, old, row| { if old != row { hunter_pet_appeared(v, shard, row) @@ -580,6 +654,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_item_instance(), "game_item_instance.insert", &view, + Some(&generation), move |v, row| item_inserted(v, &coord, shard, row), ); } @@ -589,6 +664,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_item_instance(), "game_item_instance.update", &view, + Some(&generation), move |v, old, row| item_updated(v, &coord, shard, old, row), ); } @@ -598,27 +674,42 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_item_instance(), "game_item_instance.delete", &view, + Some(&generation), move |v, row| item_deleted(v, &coord, shard, row), ); } // ---- game_world_entity ----------------------------------------------------------------- - wire_insert(db.game_world_entity(), "game_world_entity.insert", &view, move |v, row| { - entity_appeared(v, shard, row) - }); + wire_insert( + db.game_world_entity(), + "game_world_entity.insert", + &view, + Some(&generation), + move |v, row| entity_appeared(v, shard, row), + ); { // The update leg carries the live zone as well as the position: the Module re-stamps // `zone_id` on the same grid-crossing that moves the row, so a viewer's own row is the // gateway's only zone signal (`CMSG_ZONEUPDATE` is deliberately unhandled). let coord = coord.clone(); - wire_update(db.game_world_entity(), "game_world_entity.update", &view, move |v, old, new| { - entity_changed(v, shard, old, new); - zone_crossed(v, &coord, shard, new.guid, old.zone_id, new.zone_id); - }); + wire_update( + db.game_world_entity(), + "game_world_entity.update", + &view, + Some(&generation), + move |v, old, new| { + entity_changed(v, shard, old, new); + zone_crossed(v, &coord, shard, new.guid, old.zone_id, new.zone_id); + }, + ); } - wire_delete(db.game_world_entity(), "game_world_entity.delete", &view, |v, row| { - entity_vanished(v, row) - }); + wire_delete( + db.game_world_entity(), + "game_world_entity.delete", + &view, + Some(&generation), + move |v, row| entity_vanished(v, shard, row.guid, row.owner_guid), + ); // ---- game_encounter_equip --------------------------------------------------------------- // Creature virtual-item displays. The durable row feeds a late observer's peer CREATE path; @@ -627,119 +718,196 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_encounter_equip(), "game_encounter_equip.insert", &view, + Some(&generation), move |v, row| encounter_equip_changed(v, shard, row, false), ); wire_update( db.game_encounter_equip(), "game_encounter_equip.update", &view, + Some(&generation), move |v, _old, row| encounter_equip_changed(v, shard, row, false), ); wire_delete( db.game_encounter_equip(), "game_encounter_equip.delete", &view, + Some(&generation), move |v, row| encounter_equip_changed(v, shard, row, true), ); // ---- game_gameobject ------------------------------------------------------------------- - wire_insert(db.game_gameobject(), "game_gameobject.insert", &view, move |v, row| { - gameobject_appeared(v, shard, row) - }); - wire_update(db.game_gameobject(), "game_gameobject.update", &view, move |v, _old, row| { - gameobject_appeared(v, shard, row) - }); - wire_delete(db.game_gameobject(), "game_gameobject.delete", &view, |v, row| { - gameobject_vanished(v, row) - }); + wire_insert( + db.game_gameobject(), + "game_gameobject.insert", + &view, + Some(&generation), + move |v, row| gameobject_appeared(v, shard, row), + ); + wire_update( + db.game_gameobject(), + "game_gameobject.update", + &view, + Some(&generation), + move |v, _old, row| gameobject_appeared(v, shard, row), + ); + wire_delete( + db.game_gameobject(), + "game_gameobject.delete", + &view, + Some(&generation), + move |v, row| gameobject_vanished(v, shard, row.guid), + ); // ---- game_entity_motion (peer movement) ------------------------------------------------ - wire_insert(db.game_entity_motion(), "game_entity_motion.insert", &view, |v, row| { - motion(v, row) - }); - wire_update(db.game_entity_motion(), "game_entity_motion.update", &view, |v, _old, row| { - motion(v, row) - }); + wire_insert( + db.game_entity_motion(), + "game_entity_motion.insert", + &view, + Some(&generation), + |v, row| motion(v, row), + ); + wire_update( + db.game_entity_motion(), + "game_entity_motion.update", + &view, + Some(&generation), + |v, _old, row| motion(v, row), + ); // ---- game_creature_spline (creature legs) ---------------------------------------------- - wire_insert(db.game_creature_spline(), "game_creature_spline.insert", &view, |v, row| { - creature_leg(v, row) - }); - wire_update(db.game_creature_spline(), "game_creature_spline.update", &view, |v, _old, row| { - creature_leg(v, row) - }); + wire_insert( + db.game_creature_spline(), + "game_creature_spline.insert", + &view, + Some(&generation), + |v, row| creature_leg(v, row), + ); + wire_update( + db.game_creature_spline(), + "game_creature_spline.update", + &view, + Some(&generation), + |v, _old, row| creature_leg(v, row), + ); // ---- game_taxi_passenger_spline (server-owned player routes) --------------------------- wire_insert( db.game_taxi_passenger_spline(), "game_taxi_passenger_spline.insert", &view, + Some(&generation), |v, row| taxi_spline(v, row), ); wire_update( db.game_taxi_passenger_spline(), "game_taxi_passenger_spline.update", &view, + Some(&generation), |v, _old, row| taxi_spline(v, row), ); // ---- game_roll_event ------------------------------------------------------------------- // /roll broadcast. - wire_insert(db.game_roll_event(), "game_roll_event.insert", &view, move |v, row| { - roll_appeared(v, shard, row) - }); + wire_insert( + db.game_roll_event(), + "game_roll_event.insert", + &view, + Some(&generation), + move |v, row| roll_appeared(v, shard, row), + ); // ---- game_rest_state_event -------------------------------------------------------------- // Rest-state flips (zzz + blue XP bar). Self-only relay. - wire_insert(db.game_rest_state_event(), "game_rest_state_event.insert", &view, |v, row| { - rest_state_appeared(v, row) - }); + wire_insert( + db.game_rest_state_event(), + "game_rest_state_event.insert", + &view, + Some(&generation), + |v, row| rest_state_appeared(v, row), + ); // ---- game_breath_relay_event ------------------------------------------------------------ // Breath start/stop edges + server-resolved drowning combat log. Self-only relay. - wire_insert(db.game_breath_relay_event(), "game_breath_relay_event.insert", &view, |v, row| { - breath_relay_appeared(v, row) - }); + wire_insert( + db.game_breath_relay_event(), + "game_breath_relay_event.insert", + &view, + Some(&generation), + |v, row| breath_relay_appeared(v, row), + ); // ---- game_dynamic_object ----------------------------------------------------------------- // Ground-area spell visuals. Shard broadcast, instance-gated per viewer on insert; the // delete leg is ungated, matching the per-player relay byte for byte. - wire_insert(db.game_dynamic_object(), "game_dynamic_object.insert", &view, move |v, row| { - dynobj_appeared(v, shard, row) - }); - wire_delete(db.game_dynamic_object(), "game_dynamic_object.delete", &view, move |v, row| { - dynobj_vanished(v, shard, row) - }); + wire_insert( + db.game_dynamic_object(), + "game_dynamic_object.insert", + &view, + Some(&generation), + move |v, row| dynobj_appeared(v, shard, row), + ); + wire_delete( + db.game_dynamic_object(), + "game_dynamic_object.delete", + &view, + Some(&generation), + move |v, row| dynobj_vanished(v, shard, row), + ); // ---- game_corpse -------------------------------------------------------------------------- // Player corpses. Shard broadcast, instance-gated per viewer on insert/update (the // body→bones re-CREATE); the delete leg is ungated, matching the per-player relay. - wire_insert(db.game_corpse(), "game_corpse.insert", &view, move |v, row| { - corpse_appeared(v, shard, row) - }); - wire_update(db.game_corpse(), "game_corpse.update", &view, move |v, _old, row| { - corpse_changed(v, shard, row) - }); - wire_delete(db.game_corpse(), "game_corpse.delete", &view, move |v, row| { - corpse_vanished(v, shard, row) - }); + wire_insert( + db.game_corpse(), + "game_corpse.insert", + &view, + Some(&generation), + move |v, row| corpse_appeared(v, shard, row), + ); + wire_update( + db.game_corpse(), + "game_corpse.update", + &view, + Some(&generation), + move |v, _old, row| corpse_changed(v, shard, row), + ); + wire_delete( + db.game_corpse(), + "game_corpse.delete", + &view, + Some(&generation), + move |v, row| corpse_vanished(v, shard, row), + ); // ---- game_combat_event -------------------------------------------------------------------- // Melee/ranged swing log, anchored on the attacker's cell; the created-set visibility gate // runs inside the job (`combat_event_outbound`), per viewer. - wire_insert(db.game_combat_event(), "game_combat_event.insert", &view, |v, row| { - combat_event_appeared(v, row) - }); + wire_insert( + db.game_combat_event(), + "game_combat_event.insert", + &view, + Some(&generation), + move |v, row| combat_event_appeared(v, shard, row), + ); // ---- game_melee_attack -------------------------------------------------------------------- // Combat stance (ATTACKSTART/STOP + the owner-only ranged CANCEL_AUTO_REPEAT), anchored on // the attacker's indexed cell. - wire_insert(db.game_melee_attack(), "game_melee_attack.insert", &view, |v, row| { - melee_engaged(v, row) - }); - wire_delete(db.game_melee_attack(), "game_melee_attack.delete", &view, |v, row| { - melee_disengaged(v, row) - }); + wire_insert( + db.game_melee_attack(), + "game_melee_attack.insert", + &view, + Some(&generation), + move |v, row| melee_engaged(v, shard, row), + ); + wire_delete( + db.game_melee_attack(), + "game_melee_attack.delete", + &view, + Some(&generation), + move |v, row| melee_disengaged(v, shard, row), + ); // ---- recipient-keyed private event relays ----------------------------------------------- // The PRIVATE tier: every row is addressed to exactly one recipient, and on this feed the @@ -747,23 +915,36 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId // lookup) + the explicit `private_recipient_audience` predicate — never a viewer fan. // (The realm-core whisper/group twins for CROSS-shard delivery ride the same dispatchers, // armed once per realm-core connection by `arm_realm_private` below.) - wire_insert_live(db.game_resurrect_request(), "game_resurrect_request.insert", &view, |v, row| { - resurrect_offered(v, row) - }); - wire_insert_live(db.game_whisper_event(), "game_whisper_event.insert", &view, |v, row| { - whisper_appeared(v, row) - }); + wire_insert_live( + db.game_resurrect_request(), + "game_resurrect_request.insert", + &view, + Some(&generation), + |v, row| resurrect_offered(v, row), + ); + wire_insert_live( + db.game_whisper_event(), + "game_whisper_event.insert", + &view, + Some(&generation), + |v, row| whisper_appeared(v, row), + ); wire_insert_live( db.game_system_message_event(), "game_system_message_event.insert", &view, + Some(&generation), |v, row| system_message_appeared(v, row), ); { let coord = coord.clone(); - wire_insert_live(db.game_group_event(), "game_group_event.insert", &view, move |v, row| { - group_event_appeared(v, &coord, row) - }); + wire_insert_live( + db.game_group_event(), + "game_group_event.insert", + &view, + Some(&generation), + move |v, row| group_event_appeared(v, &coord, row), + ); } { let insert_coord = coord.clone(); @@ -771,6 +952,7 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_group_member(), "game_group_member.insert", &view, + Some(&generation), move |v, row| group_member_mirror_changed(v, &insert_coord, row), ); let delete_coord = coord.clone(); @@ -778,17 +960,26 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId db.game_group_member(), "game_group_member.delete", &view, + Some(&generation), move |v, row| group_member_mirror_changed(v, &delete_coord, row), ); } - wire_insert_live(db.game_trade_event(), "game_trade_event.insert", &view, |v, row| { - trade_event_appeared(v, row) - }); + wire_insert_live( + db.game_trade_event(), + "game_trade_event.insert", + &view, + Some(&generation), + |v, row| trade_event_appeared(v, row), + ); { let coord = coord.clone(); - wire_insert_live(db.game_duel_event(), "game_duel_event.insert", &view, move |v, row| { - duel_event_appeared(v, &coord, row) - }); + wire_insert_live( + db.game_duel_event(), + "game_duel_event.insert", + &view, + Some(&generation), + move |v, row| duel_event_appeared(v, &coord, row), + ); } // ---- game_aura ------------------------------------------------------------------------------ @@ -800,33 +991,42 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId { let view = view.clone(); let coord = coord.clone(); + let generation = generation.clone(); db.game_aura().on_insert(move |_ctx, row| { guarded("game_aura.insert", || { - view.auras.upsert(shard, row); - let n = view.auras.stealth_count(row.target_guid); - aura_applied(&view, &coord, row, n) + ShardGeneration::run(Some(&generation), || { + view.auras.upsert(shard, row); + let n = view.auras.stealth_count(shard, row.target_guid); + aura_applied(&view, &coord, shard, row, n) + }) }); }); } { let view = view.clone(); let coord = coord.clone(); + let generation = generation.clone(); db.game_aura().on_update(move |_ctx, old, row| { let expires_changed = old.expires_at != row.expires_at; guarded("game_aura.update", || { - view.auras.upsert(shard, row); - aura_updated(&view, &coord, row, expires_changed) + ShardGeneration::run(Some(&generation), || { + view.auras.upsert(shard, row); + aura_updated(&view, &coord, shard, row, expires_changed) + }) }); }); } { let view = view.clone(); let coord = coord.clone(); + let generation = generation.clone(); db.game_aura().on_delete(move |_ctx, row| { guarded("game_aura.delete", || { - view.auras.remove(shard, row); - let n = view.auras.stealth_count(row.target_guid); - aura_removed(&view, &coord, row, n) + ShardGeneration::run(Some(&generation), || { + view.auras.remove(shard, row); + let n = view.auras.stealth_count(shard, row.target_guid); + aura_removed(&view, &coord, shard, row, n) + }) }); }); } @@ -834,12 +1034,20 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId // ---- game_spell_cast_event / game_spell_impact_event -------------------------------------- // Cast visuals (the full cast-lock contract, per viewer) + deferred projectile impact logs, // both anchored on the caster's cell. - wire_insert(db.game_spell_cast_event(), "game_spell_cast_event.insert", &view, |v, row| { - cast_event_appeared(v, row) - }); - wire_insert(db.game_spell_impact_event(), "game_spell_impact_event.insert", &view, |v, row| { - impact_appeared(v, row) - }); + wire_insert( + db.game_spell_cast_event(), + "game_spell_cast_event.insert", + &view, + Some(&generation), + move |v, row| cast_event_appeared(v, shard, row), + ); + wire_insert( + db.game_spell_impact_event(), + "game_spell_impact_event.insert", + &view, + Some(&generation), + move |v, row| impact_appeared(v, shard, row), + ); // ---- game_emote_event / game_chat_event / game_channel_event ------------------------------ // The social tier. Emotes and say/yell are range-bound, so both are anchored on the sender's @@ -847,45 +1055,71 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId // per viewer inside the job. { let coord = coord.clone(); - wire_insert(db.game_emote_event(), "game_emote_event.insert", &view, move |v, row| { - emote_appeared(v, &coord, row) - }); + wire_insert( + db.game_emote_event(), + "game_emote_event.insert", + &view, + Some(&generation), + move |v, row| emote_appeared(v, &coord, shard, row), + ); } { let coord = coord.clone(); - wire_insert(db.game_chat_event(), "game_chat_event.insert", &view, move |v, row| { - chat_appeared(v, &coord, row) - }); + wire_insert( + db.game_chat_event(), + "game_chat_event.insert", + &view, + Some(&generation), + move |v, row| chat_appeared(v, &coord, shard, row), + ); } { let coord = coord.clone(); - wire_insert(db.game_channel_event(), "game_channel_event.insert", &view, move |v, row| { - channel_appeared(v, &coord, shard, row) - }); + wire_insert( + db.game_channel_event(), + "game_channel_event.insert", + &view, + Some(&generation), + move |v, row| channel_appeared(v, &coord, shard, row), + ); } // ---- game_zone_weather ----------------------------------------------------------------------- // The live sky. ONE callback per shard, routed by the viewer's stored zone — there are no // per-character weather subscriptions. Insert and update run the same body: the first row a // zone ever gets is as much a visible change as every later one. - wire_insert(db.game_zone_weather(), "game_zone_weather.insert", &view, move |v, row| { - weather_changed(v, shard, row) - }); - wire_update(db.game_zone_weather(), "game_zone_weather.update", &view, move |v, _old, row| { - weather_changed(v, shard, row) - }); + wire_insert( + db.game_zone_weather(), + "game_zone_weather.insert", + &view, + Some(&generation), + move |v, row| weather_changed(v, shard, row), + ); + wire_update( + db.game_zone_weather(), + "game_zone_weather.update", + &view, + Some(&generation), + move |v, _old, row| weather_changed(v, shard, row), + ); // ---- game_player_skill ---------------------------------------------------------------------- // Live skill pane. Self-only relay; insert (line learned) and update (skill-up) run the same // body against the viewer's own slot map. - wire_insert(db.game_player_skill(), "game_player_skill.insert", &view, |v, row| { - skill_changed(v, row) - }); - wire_update(db.game_player_skill(), "game_player_skill.update", &view, |v, _old, row| { - skill_changed(v, row) - }); - - drop(guard); + wire_insert( + db.game_player_skill(), + "game_player_skill.insert", + &view, + Some(&generation), + |v, row| skill_changed(v, row), + ); + wire_update( + db.game_player_skill(), + "game_player_skill.update", + &view, + Some(&generation), + |v, _old, row| skill_changed(v, row), + ); } /// Register the cross-shard PRIVATE-tier twins (#22 → #483) on the REALM-CORE connection: whisper @@ -905,12 +1139,20 @@ pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId pub(crate) fn arm_realm_private(view: Arc, realm: Coordinator, coord: Coordinator) { let guard = realm.0.coord(); let db = &guard.conn.db; - wire_insert_live(db.game_whisper_event(), "realm.game_whisper_event.insert", &view, |v, row| { - whisper_appeared(v, row) - }); - wire_insert_live(db.game_group_event(), "realm.game_group_event.insert", &view, move |v, row| { - group_event_appeared(v, &coord, row) - }); + wire_insert_live( + db.game_whisper_event(), + "realm.game_whisper_event.insert", + &view, + None, + |v, row| whisper_appeared(v, row), + ); + wire_insert_live( + db.game_group_event(), + "realm.game_group_event.insert", + &view, + None, + move |v, row| group_event_appeared(v, &coord, row), + ); } /// Run one shared-connection callback body with a panic firewall. On a per-player connection a @@ -940,13 +1182,17 @@ fn wire_insert( table: T, label: &'static str, view: &Arc, + generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); + let generation = generation.cloned(); table.on_insert(move |_ctx, row| { - guarded(label, || f(&view, row)); + guarded(label, || { + ShardGeneration::run(generation.as_ref(), || f(&view, row)) + }); }); } @@ -964,16 +1210,20 @@ fn wire_insert_live( table: T, label: &'static str, view: &Arc, + generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); + let generation = generation.cloned(); table.on_insert(move |ctx, row| { if is_initial_apply(&ctx.event) { return; } - guarded(label, || f(&view, row)); + guarded(label, || { + ShardGeneration::run(generation.as_ref(), || f(&view, row)) + }); }); } @@ -982,13 +1232,17 @@ fn wire_update( table: T, label: &'static str, view: &Arc, + generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row, &T::Row) + Send + 'static, ) where T: TableWithPrimaryKey, { let view = view.clone(); + let generation = generation.cloned(); table.on_update(move |_ctx, old, new| { - guarded(label, || f(&view, old, new)); + guarded(label, || { + ShardGeneration::run(generation.as_ref(), || f(&view, old, new)) + }); }); } @@ -997,13 +1251,17 @@ fn wire_delete( table: T, label: &'static str, view: &Arc, + generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); + let generation = generation.cloned(); table.on_delete(move |_ctx, row| { - guarded(label, || f(&view, row)); + guarded(label, || { + ShardGeneration::run(generation.as_ref(), || f(&view, row)) + }); }); } @@ -1024,10 +1282,15 @@ fn entity_key(row: &WorldEntity) -> CellKey { /// its cell. fn entity_appeared(view: &Arc, shard: ShardId, row: &WorldEntity) { let key = entity_key(row); - view.spatial - .upsert_entity(EntityLayer::WorldEntity, row.guid, key, shard); + view.spatial.upsert_entity( + EntityLayer::WorldEntity, + row.guid, + key, + shard, + row.owner_guid, + ); let row = Arc::new(row.clone()); - for session in view.world_entity_recipients(row.guid, key) { + for session in view.world_entity_recipients(shard, row.guid, key) { offer_create_job(view, shard, session, &row); } } @@ -1041,10 +1304,14 @@ fn entity_appeared(view: &Arc, shard: ShardId, row: &WorldEntity) { /// * nobody else is touched. fn entity_changed(view: &Arc, shard: ShardId, old: &WorldEntity, new: &WorldEntity) { let key = entity_key(new); - let previous = view - .spatial - .upsert_entity(EntityLayer::WorldEntity, new.guid, key, shard); - let new_recipients = view.world_entity_recipients(new.guid, key); + let previous = view.spatial.upsert_entity( + EntityLayer::WorldEntity, + new.guid, + key, + shard, + new.owner_guid, + ); + let new_recipients = view.world_entity_recipients(shard, new.guid, key); if let Some(old_key) = previous { if old_key != key { let still: HashSet = new_recipients.iter().copied().collect(); @@ -1063,16 +1330,15 @@ fn entity_changed(view: &Arc, shard: ShardId, old: &WorldEntity, new: } /// The row is gone from the database. Everyone who could see it gets a DESTROY. -fn entity_vanished(view: &WorldView, row: &WorldEntity) { - // Take the cell the index remembers rather than recomputing from the row: a delete's payload is - // the LAST known row, and if a move and a delete land in the same transaction the index is the - // one that knows where the viewers were told it was. - let key = view +fn entity_vanished(view: &WorldView, shard: ShardId, guid: u64, owner_guid: u64) { + let Some((key, _)) = view .spatial - .remove_entity(EntityLayer::WorldEntity, row.guid) - .unwrap_or_else(|| entity_key(row)); - for session in view.world_entity_recipients(row.guid, key) { - destroy_job(view, session, row.guid, row.owner_guid); + .remove_entity(EntityLayer::WorldEntity, guid, shard) + else { + return; + }; + for session in view.world_entity_recipients(shard, guid, key) { + destroy_job(view, session, guid, owner_guid); } } @@ -1093,7 +1359,7 @@ fn encounter_equip_changed(view: &WorldView, shard: ShardId, row: &EncounterEqui }; let key = entity_key(&entity); let row = Arc::new(row.clone()); - for session in view.world_entity_recipients(row.creature_guid, key) { + for session in view.world_entity_recipients(shard, row.creature_guid, key) { let Some(viewer) = view.viewer(session) else { continue; }; @@ -1120,7 +1386,7 @@ fn offer_create_job( let (view, row) = (view.clone(), row.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { - super::subscriptions::offer_peer_create_for(&coord, &view, &viewer, &row) + super::subscriptions::offer_peer_create_for(&coord, &view, shard, &viewer, &row) }); } @@ -1140,7 +1406,7 @@ fn update_job( let (view, old, new) = (view.clone(), old.clone(), new.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { - super::subscriptions::relay_entity_update(&coord, &view, &viewer, &old, &new) + super::subscriptions::relay_entity_update(&coord, &view, shard, &viewer, &old, &new) }); } @@ -1162,20 +1428,18 @@ fn gameobject_appeared(view: &WorldView, shard: ShardId, row: &GameObject) { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); let previous = view .spatial - .upsert_entity(EntityLayer::GameObject, row.guid, key, shard); - let recipients = view.spatial.viewers_of(EntityLayer::GameObject, key); + .upsert_entity(EntityLayer::GameObject, row.guid, key, shard, 0); + let recipients = view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]); if let Some(old_key) = previous { if old_key != key { - let still: HashSet = recipients.iter().copied().collect(); - for session in view.spatial.viewers_of(EntityLayer::GameObject, old_key) { - if !still.contains(&session) { - if let Some(viewer) = view.viewer(session) { - let tx = viewer.tx.clone(); - let guid = row.guid; - enqueue(&tx, move || { - super::subscriptions::relay_peer_destroy(&viewer, guid, 0) - }); - } + let still: HashSet = recipients.iter().map(|v| v.session).collect(); + for viewer in view.cell_audience(shard, Some(old_key), BOX_HALF_SPAN, &[]) { + if !still.contains(&viewer.session) { + let tx = viewer.tx.clone(); + let guid = row.guid; + enqueue(&tx, move || { + super::subscriptions::relay_gameobject_destroy(&viewer, guid) + }); } } } @@ -1184,10 +1448,7 @@ fn gameobject_appeared(view: &WorldView, shard: ShardId, row: &GameObject) { return; }; let row = Arc::new(row.clone()); - for session in recipients { - let Some(viewer) = view.viewer(session) else { - continue; - }; + for viewer in recipients { let (coord, row) = (coord.clone(), row.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1196,17 +1457,15 @@ fn gameobject_appeared(view: &WorldView, shard: ShardId, row: &GameObject) { } } -fn gameobject_vanished(view: &WorldView, row: &GameObject) { - let key = view +fn gameobject_vanished(view: &WorldView, shard: ShardId, guid: u64) { + let Some((key, _)) = view .spatial - .remove_entity(EntityLayer::GameObject, row.guid) - .unwrap_or_else(|| CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y)); - for session in view.spatial.viewers_of(EntityLayer::GameObject, key) { - let Some(viewer) = view.viewer(session) else { - continue; - }; + .remove_entity(EntityLayer::GameObject, guid, shard) + else { + return; + }; + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { let tx = viewer.tx.clone(); - let guid = row.guid; // DESTROY stays ungated (the `on_melee_delete` precedent): SMSG_DESTROY_OBJECT for a guid // the client never created is a client no-op, and gating it risks a stale object. enqueue(&tx, move || { @@ -1744,9 +2003,10 @@ fn cells_within(range_yd: f32) -> i32 { (range_yd / GRID_CELL_SIZE).ceil() as i32 + 1 } -fn combat_audience(view: &WorldView, row: &CombatEvent) -> Vec> { +fn combat_audience(view: &WorldView, shard: ShardId, row: &CombatEvent) -> Vec> { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); view.cell_audience( + shard, Some(key), BOX_HALF_SPAN, &[row.attacker_guid, row.target_guid], @@ -1756,9 +2016,9 @@ fn combat_audience(view: &WorldView, row: &CombatEvent) -> Vec> { /// A swing landed → the full 097 packet fork (`combat_event_outbound`: ranged GO + delayed damage /// log / melee ATTACKERSTATEUPDATE / killing-blow ATTACKSTOP) per viewer, gated on that viewer's /// `created` set inside the job. -fn combat_event_appeared(view: &WorldView, row: &CombatEvent) { +fn combat_event_appeared(view: &WorldView, shard: ShardId, row: &CombatEvent) { let row = Arc::new(row.clone()); - for viewer in combat_audience(view, &row) { + for viewer in combat_audience(view, shard, &row) { let row = row.clone(); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -1768,18 +2028,18 @@ fn combat_event_appeared(view: &WorldView, row: &CombatEvent) { } /// An engagement row names no cell; the attacker's indexed cell anchors it. -fn melee_audience(view: &WorldView, row: &MeleeAttack) -> Vec> { +fn melee_audience(view: &WorldView, shard: ShardId, row: &MeleeAttack) -> Vec> { let key = view .spatial - .entity_cell(EntityLayer::WorldEntity, row.attacker_guid); - view.cell_audience(key, BOX_HALF_SPAN, &[row.attacker_guid, row.target_guid]) + .entity_cell_on_shard(EntityLayer::WorldEntity, row.attacker_guid, shard); + view.cell_audience(shard, key, BOX_HALF_SPAN, &[row.attacker_guid, row.target_guid]) } /// A melee/ranged engagement began → SMSG_ATTACKSTART per viewer (`melee_engage_outbound` runs /// the ranged skip + created gate in the job). -fn melee_engaged(view: &WorldView, row: &MeleeAttack) { +fn melee_engaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { let row = Arc::new(row.clone()); - for viewer in melee_audience(view, &row) { + for viewer in melee_audience(view, shard, &row) { let row = row.clone(); let created = viewer.created.clone(); let tx = viewer.tx.clone(); @@ -1791,9 +2051,9 @@ fn melee_engaged(view: &WorldView, row: &MeleeAttack) { /// An engagement row was removed → ATTACKSTOP / the owner-only CANCEL_AUTO_REPEAT /// (`melee_disengage_outbound`, keyed on each viewer's own guid). -fn melee_disengaged(view: &WorldView, row: &MeleeAttack) { +fn melee_disengaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { let row = Arc::new(row.clone()); - for viewer in melee_audience(view, &row) { + for viewer in melee_audience(view, shard, &row) { let row = row.clone(); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -1964,24 +2224,31 @@ fn group_member_mirror_changed(view: &WorldView, coord: &Coordinator, row: &Grou /// An aura names its target and nothing else; the target's indexed cell anchors it. A target /// with no row (a unit already gone) leaves the owner leg as the only recipient, and the owner /// is the only viewer whose gate could still pass. -fn aura_audience(view: &WorldView, row: &Aura) -> Vec> { +fn aura_audience(view: &WorldView, shard: ShardId, row: &Aura) -> Vec> { let key = view .spatial - .entity_cell(EntityLayer::WorldEntity, row.target_guid); - view.cell_audience(key, BOX_HALF_SPAN, &[row.target_guid]) + .entity_cell_on_shard(EntityLayer::WorldEntity, row.target_guid, shard); + view.cell_audience(shard, key, BOX_HALF_SPAN, &[row.target_guid]) } /// An aura appeared → per viewer: array sync + self-only packets + the stealth HIDE transition /// (`aura_insert_outbound`, all gates inside the job against that viewer's own state). -fn aura_applied(view: &Arc, coord: &Coordinator, row: &Aura, stealth_count: usize) { +fn aura_applied( + view: &Arc, + coord: &Coordinator, + shard: ShardId, + row: &Aura, + stealth_count: usize, +) { let row = Arc::new(row.clone()); - for viewer in aura_audience(view, &row) { + for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { super::subscriptions::aura_insert_outbound( &coord, &view, + shard, viewer.session, &viewer.created, viewer.self_guid, @@ -1993,15 +2260,16 @@ fn aura_applied(view: &Arc, coord: &Coordinator, row: &Aura, stealth_ } /// An aura changed → per viewer: array sync + refresh-only duration + self-only packets. -fn aura_updated(view: &Arc, coord: &Coordinator, row: &Aura, expires_changed: bool) { +fn aura_updated(view: &Arc, coord: &Coordinator, shard: ShardId, row: &Aura, expires_changed: bool) { let row = Arc::new(row.clone()); - for viewer in aura_audience(view, &row) { + for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { super::subscriptions::aura_update_outbound( &coord, &view, + shard, &viewer.created, viewer.self_guid, &row, @@ -2013,15 +2281,22 @@ fn aura_updated(view: &Arc, coord: &Coordinator, row: &Aura, expires_ /// An aura expired/was removed → per viewer: array sync + self-only packets + the stealth REVEAL /// transition. -fn aura_removed(view: &Arc, coord: &Coordinator, row: &Aura, stealth_count: usize) { +fn aura_removed( + view: &Arc, + coord: &Coordinator, + shard: ShardId, + row: &Aura, + stealth_count: usize, +) { let row = Arc::new(row.clone()); - for viewer in aura_audience(view, &row) { + for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { super::subscriptions::aura_delete_outbound( &coord, &view, + shard, viewer.session, &viewer.created, viewer.self_guid, @@ -2032,9 +2307,10 @@ fn aura_removed(view: &Arc, coord: &Coordinator, row: &Aura, stealth_ } } -fn cast_audience(view: &WorldView, row: &SpellCastEvent) -> Vec> { +fn cast_audience(view: &WorldView, shard: ShardId, row: &SpellCastEvent) -> Vec> { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); view.cell_audience( + shard, Some(key), BOX_HALF_SPAN, &[row.caster_guid, row.target_guid], @@ -2043,9 +2319,9 @@ fn cast_audience(view: &WorldView, row: &SpellCastEvent) -> Vec> { /// A cast event landed → the full cast-lock sequence per viewer (`cast_event_outbound`: the /// caster-private branches key on each viewer's own guid inside the job). -fn cast_event_appeared(view: &WorldView, row: &SpellCastEvent) { +fn cast_event_appeared(view: &WorldView, shard: ShardId, row: &SpellCastEvent) { let row = Arc::new(row.clone()); - for viewer in cast_audience(view, &row) { + for viewer in cast_audience(view, shard, &row) { let row = row.clone(); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -2055,9 +2331,10 @@ fn cast_event_appeared(view: &WorldView, row: &SpellCastEvent) { } } -fn impact_audience(view: &WorldView, row: &SpellImpactEvent) -> Vec> { +fn impact_audience(view: &WorldView, shard: ShardId, row: &SpellImpactEvent) -> Vec> { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); view.cell_audience( + shard, Some(key), BOX_HALF_SPAN, &[row.caster_guid, row.target_guid], @@ -2065,9 +2342,9 @@ fn impact_audience(view: &WorldView, row: &SpellImpactEvent) -> Vec> } /// A projectile impact landed → the floating damage number per viewer that can see the caster. -fn impact_appeared(view: &WorldView, row: &SpellImpactEvent) { +fn impact_appeared(view: &WorldView, shard: ShardId, row: &SpellImpactEvent) { let row = Arc::new(row.clone()); - for viewer in impact_audience(view, &row) { + for viewer in impact_audience(view, shard, &row) { let row = row.clone(); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -2076,9 +2353,10 @@ fn impact_appeared(view: &WorldView, row: &SpellImpactEvent) { } } -fn emote_audience(view: &WorldView, row: &EmoteEvent) -> Vec> { +fn emote_audience(view: &WorldView, shard: ShardId, row: &EmoteEvent) -> Vec> { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); view.cell_audience( + shard, Some(key), BOX_HALF_SPAN, &[row.sender_guid, row.target_guid], @@ -2087,9 +2365,9 @@ fn emote_audience(view: &WorldView, row: &EmoteEvent) -> Vec> { /// An emote landed → SMSG_TEXT_EMOTE + SMSG_EMOTE per viewer that can see the sender. Name /// resolve runs in the job against `coord`'s cache. -fn emote_appeared(view: &WorldView, coord: &Coordinator, row: &EmoteEvent) { +fn emote_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &EmoteEvent) { let row = Arc::new(row.clone()); - for viewer in emote_audience(view, &row) { + for viewer in emote_audience(view, shard, &row) { let (row, coord) = (row.clone(), coord.clone()); let tx = viewer.tx.clone(); enqueue(&tx, move || { @@ -2100,20 +2378,20 @@ fn emote_appeared(view: &WorldView, coord: &Coordinator, row: &EmoteEvent) { /// A chat line names its speaker and nothing else; the speaker's indexed cell anchors it, widened /// to the kind's yard range (a yell carries 300 yd, four cells past the box). -fn chat_audience(view: &WorldView, row: &ChatEvent) -> Vec> { +fn chat_audience(view: &WorldView, shard: ShardId, row: &ChatEvent) -> Vec> { let key = view .spatial - .entity_cell(EntityLayer::WorldEntity, row.sender_guid); + .entity_cell_on_shard(EntityLayer::WorldEntity, row.sender_guid, shard); let span = cells_within(super::subscriptions::chat_range_yd(row.chat_type)); - view.cell_audience(key, span, &[row.sender_guid, row.target_guid]) + view.cell_audience(shard, key, span, &[row.sender_guid, row.target_guid]) } /// A say/yell line landed → SMSG_MESSAGECHAT per viewer, `chat_in_range`-gated inside the job /// (`chat_event_outbound`: speaker echo, SAY/YELL radii, map+instance fence, coordinator-cache /// endpoint lookups). -fn chat_appeared(view: &WorldView, coord: &Coordinator, row: &ChatEvent) { +fn chat_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &ChatEvent) { let row = Arc::new(row.clone()); - for viewer in chat_audience(view, &row) { + for viewer in chat_audience(view, shard, &row) { let (row, coord) = (row.clone(), coord.clone()); let self_guid = viewer.self_guid; let tx = viewer.tx.clone(); @@ -2253,7 +2531,7 @@ mod family_audience_tests { #[test] fn membership_mirror_drives_viewer_relative_loot_tag_flags() { let source = include_str!("world_view.rs"); - let arm = crate::test_scan::code_of(source, "pub(crate) fn arm_shard"); + let arm = crate::test_scan::code_of(source, "fn register_shard_callbacks"); let arm: String = arm.split_whitespace().collect(); assert!(arm.contains("wire_insert(db.game_group_member()")); assert!(arm.contains("wire_delete(db.game_group_member()")); @@ -2608,9 +2886,9 @@ mod family_audience_tests { // both row tables must appear once in each independent layer delta. let entering = CellKey::at(0, 0, 3, 0); view.spatial - .upsert_entity(EntityLayer::WorldEntity, 77, entering, 0); + .upsert_entity(EntityLayer::WorldEntity, 77, entering, 0, 0); view.spatial - .upsert_entity(EntityLayer::GameObject, 77, entering, 1); + .upsert_entity(EntityLayer::GameObject, 77, entering, 1, 0); let delta = view .spatial .move_viewer_delta(viewer.session, CellKey::at(0, 0, 1, 0)) @@ -2620,7 +2898,7 @@ mod family_audience_tests { // Removing one table's row cannot erase the equal-valued GUID in the other table. assert_eq!( - view.spatial.remove_entity(EntityLayer::WorldEntity, 77), + view.spatial.remove_entity(EntityLayer::WorldEntity, 77, 0), Some(entering) ); assert!(view @@ -3187,16 +3465,16 @@ mod family_audience_tests { .viewers_of(EntityLayer::WorldEntity, far) .is_empty()); assert_eq!( - view.world_entity_recipients(viewer.self_guid, far), + view.world_entity_recipients(0, viewer.self_guid, far), vec![viewer.session] ); assert_eq!( - view.world_entity_recipients(viewer.self_guid, anchor), + view.world_entity_recipients(0, viewer.self_guid, anchor), vec![viewer.session] ); view.remove_viewer(viewer.session); assert!(view - .world_entity_recipients(viewer.self_guid, far) + .world_entity_recipients(0, viewer.self_guid, far) .is_empty()); } @@ -3212,7 +3490,7 @@ mod family_audience_tests { "a recenter must enqueue world-entity visibility work before game-object work" ); - let seed_start = source.rfind("pub(crate) fn seed_from_caches(view:").unwrap(); + let seed_start = source.rfind("fn seed_shard_from_cache(view:").unwrap(); let sweep = &source[sweep_start..seed_start]; assert!( sweep.find("EntityLayer::WorldEntity").unwrap() @@ -3289,7 +3567,7 @@ pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, }; let (view, viewer, row) = (view.clone(), viewer.clone(), Arc::new(row)); enqueue(&viewer.tx.clone(), move || { - super::subscriptions::offer_peer_create_for(&coord, &view, &viewer, &row) + super::subscriptions::offer_peer_create_for(&coord, &view, shard, &viewer, &row) }); } } @@ -3340,7 +3618,7 @@ pub(crate) fn sweep_into_view(view: &WorldView, viewer: &Arc) { else { continue; }; - for o in super::subscriptions::offer_peer_create_for(&coord, view, viewer, &row) { + for o in super::subscriptions::offer_peer_create_for(&coord, view, shard, viewer, &row) { let _ = viewer.tx.send(o); } } @@ -3372,35 +3650,92 @@ pub(crate) fn sweep_into_view(view: &WorldView, viewer: &Arc) { } } -/// Seed the index from the coordinator caches at startup — the rows that were already resident when -/// the gateway connected never fire `on_insert`, so without this the world is invisible until every -/// entity happens to move. -pub(crate) fn seed_from_caches(view: &WorldView) { - let shards = view.shards.read().unwrap().clone(); - let mut auras = Vec::new(); - for (id, coord) in shards.iter().enumerate() { - let guard = coord.0.coord(); - for row in guard.conn.db.game_world_entity().iter() { - view.spatial - .upsert_entity(EntityLayer::WorldEntity, row.guid, entity_key(&row), id); - } - for row in guard.conn.db.game_gameobject().iter() { - view.spatial.upsert_entity( - EntityLayer::GameObject, +/// Replace this shard's rows on its pump. No SDK message or old-generation callback can +/// interleave with the snapshot, and other shards retain their current rows. +fn seed_shard_from_cache(view: &Arc, shard: ShardId, db: &RemoteTables) { + reconcile_shard( + view, + shard, + db.game_world_entity().iter().collect(), + db.game_gameobject().iter().collect(), + db.game_aura().iter().collect(), + ); +} + +fn reconcile_shard( + view: &Arc, + shard: ShardId, + entities: Vec, + objects: Vec, + auras: Vec, +) { + let old_auras = view.auras.replace_shard(shard, auras.iter().cloned()); + let removed_entities = view.spatial.replace_shard( + EntityLayer::WorldEntity, + shard, + entities + .iter() + .map(|row| (row.guid, entity_key(row), row.owner_guid)), + ); + let removed_objects = view.spatial.replace_shard( + EntityLayer::GameObject, + shard, + objects.iter().map(|row| { + ( row.guid, CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y), - id, - ); + 0, + ) + }), + ); + for (layer, removed) in [ + (EntityLayer::WorldEntity, removed_entities), + (EntityLayer::GameObject, removed_objects), + ] { + for (guid, key, owner_guid) in removed { + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { + if view.spatial.can_see(layer, viewer.session, guid) { + continue; + } + let tx = viewer.tx.clone(); + enqueue(&tx, move || match layer { + EntityLayer::WorldEntity => { + super::subscriptions::relay_peer_destroy(&viewer, guid, owner_guid) + } + EntityLayer::GameObject => { + super::subscriptions::relay_gameobject_destroy(&viewer, guid) + } + }); + } + } + } + if let Some(coord) = view.shard(shard) { + for row in old_auras { + let count = view.auras.stealth_count(shard, row.target_guid); + aura_removed(view, &coord, shard, &row, count); + } + for row in auras { + let count = view.auras.stealth_count(shard, row.target_guid); + aura_applied(view, &coord, shard, &row, count); + } + } + // Existing viewers keep their CREATE dedup state. Newly resident rows are offered normally. + for row in entities { + if view.spatial.shard_of(EntityLayer::WorldEntity, row.guid) != Some(shard) { + continue; + } + let row = Arc::new(row); + for session in view.world_entity_recipients(shard, row.guid, entity_key(&row)) { + offer_create_job(view, shard, session, &row); + } + } + for row in objects { + if view.spatial.shard_of(EntityLayer::GameObject, row.guid) == Some(shard) { + gameobject_appeared(view, shard, &row); } - auras.extend(guard.conn.db.game_aura().iter().map(|row| (id, row))); } - view.auras.replace_all(auras); - let (entities, _, cells) = view.spatial.stats(EntityLayer::WorldEntity); - let (gos, _, go_cells) = view.spatial.stats(EntityLayer::GameObject); let (aura_rows, aura_units) = view.auras.stats(); log::info!( - "shared AOI index seeded from {} shard cache(s): {entities} entities in {cells} cells, \ - {gos} gameobjects in {go_cells} cells, {aura_rows} auras on {aura_units} units", - view.shard_count() + "shared AOI shard {shard} reconciled; {aura_rows} auras on {aura_units} shard-local units" ); } diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index 3a373638b..8f6ab294d 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -66,7 +66,7 @@ fn realm(rng: &mut Rng, players: usize, creatures: usize, span_yd: f32) -> Realm let (x, y) = position(rng); let key = CellKey::of_position(0, 0, x, y); view.spatial - .upsert_entity(EntityLayer::WorldEntity, guid, key, 0); + .upsert_entity(EntityLayer::WorldEntity, guid, key, 0, 0); let (tx, rx) = SessionTx::with_depth(0); let v = viewer(view.next_session_id(), guid, tx); view.add_viewer_on_shard(v.clone(), key, 0); @@ -83,6 +83,7 @@ fn realm(rng: &mut Rng, players: usize, creatures: usize, span_yd: f32) -> Realm guid, CellKey::of_position(0, 0, x, y), 0, + 0, ); positions.insert(guid, (x, y)); guid @@ -335,8 +336,8 @@ fn combat_candidates_gate_to_exactly_the_whole_shard_recipients() { let row = combat(&realm, attacker, target); let gate = |v: &Viewer| created_contains(v, attacker); - let candidates = sessions(combat_audience(&realm.view, &row)); - combat_event_appeared(&realm.view, &row); + let candidates = sessions(combat_audience(&realm.view, 0, &row)); + combat_event_appeared(&realm.view, 0, &row); assert_eq!( realm.jobs_landed(), candidates, @@ -369,10 +370,10 @@ fn melee_candidates_gate_to_exactly_the_whole_shard_recipients() { let row = melee(attacker, realm.actor(&mut rng)); let gate = |v: &Viewer| created_contains(v, attacker); - let candidates = sessions(melee_audience(&realm.view, &row)); - melee_engaged(&realm.view, &row); + let candidates = sessions(melee_audience(&realm.view, 0, &row)); + melee_engaged(&realm.view, 0, &row); assert_eq!(realm.jobs_landed(), candidates); - melee_disengaged(&realm.view, &row); + melee_disengaged(&realm.view, 0, &row); assert_eq!(realm.jobs_landed(), candidates); let gated: HashSet = realm @@ -405,7 +406,7 @@ fn aura_candidates_gate_to_exactly_the_whole_shard_recipients() { .can_see(EntityLayer::WorldEntity, v.session, target) }; - let candidates = sessions(aura_audience(&realm.view, &row)); + let candidates = sessions(aura_audience(&realm.view, 0, &row)); let gated: HashSet = realm .viewers .iter() @@ -428,17 +429,17 @@ fn an_aura_on_an_unindexed_unit_reaches_its_owner_only() { realm .view .spatial - .remove_entity(EntityLayer::WorldEntity, player); + .remove_entity(EntityLayer::WorldEntity, player, 0); assert_eq!( - sessions(aura_audience(&realm.view, &aura(1, player, 0xA0))), + sessions(aura_audience(&realm.view, 0, &aura(1, player, 0xA0))), HashSet::from([realm.viewers[3].session]) ); let creature = realm.creatures[0]; realm .view .spatial - .remove_entity(EntityLayer::WorldEntity, creature); - assert!(aura_audience(&realm.view, &aura(2, creature, 0xA0)).is_empty()); + .remove_entity(EntityLayer::WorldEntity, creature, 0); + assert!(aura_audience(&realm.view, 0, &aura(2, creature, 0xA0)).is_empty()); } // =============================================================================================== @@ -471,25 +472,25 @@ fn cast_impact_and_emote_reach_the_actors_box_and_both_owners() { let row = cast(&realm, actor, target); assert_eq!( - sessions(cast_audience(&realm.view, &row)), + sessions(cast_audience(&realm.view, 0, &row)), want, "cast by {actor:#x}" ); - cast_event_appeared(&realm.view, &row); + cast_event_appeared(&realm.view, 0, &row); assert_eq!(realm.jobs_landed(), want); let row = impact(&realm, actor, target); assert_eq!( - sessions(impact_audience(&realm.view, &row)), + sessions(impact_audience(&realm.view, 0, &row)), want, "impact by {actor:#x}" ); - impact_appeared(&realm.view, &row); + impact_appeared(&realm.view, 0, &row); assert_eq!(realm.jobs_landed(), want); let row = emote(&realm, actor, target); assert_eq!( - sessions(emote_audience(&realm.view, &row)), + sessions(emote_audience(&realm.view, 0, &row)), want, "emote by {actor:#x}" ); @@ -526,7 +527,7 @@ fn say_and_yell_candidates_gate_to_exactly_the_whole_shard_listeners() { let row = chat(speaker, chat_type); let gate = |v: &Viewer| hears(&realm, speaker, v, chat_type); - let candidates = sessions(chat_audience(&realm.view, &row)); + let candidates = sessions(chat_audience(&realm.view, 0, &row)); let gated: HashSet = realm .viewers .iter() @@ -560,9 +561,9 @@ fn chat_from_an_unindexed_speaker_reaches_the_speaker_only() { realm .view .spatial - .remove_entity(EntityLayer::WorldEntity, speaker); + .remove_entity(EntityLayer::WorldEntity, speaker, 0); assert_eq!( - sessions(chat_audience(&realm.view, &chat(speaker, YELL))), + sessions(chat_audience(&realm.view, 0, &chat(speaker, YELL))), HashSet::from([realm.viewers[0].session]) ); } @@ -577,20 +578,20 @@ fn aura_index_insert_update_delete_by_shard_and_id() { index.upsert(0, &aura(1, 77, A_STEALTH)); index.upsert(0, &aura(2, 77, 0xA0)); index.upsert(1, &aura(1, 78, 0xA0)); // shard 1 reuses id 1 for another unit - assert_eq!(index.stealth_count(77), 1); - assert!(index.is_stealthed(77)); - assert!(!index.is_stealthed(78)); - assert_eq!(index.on_target(77).len(), 2); + assert_eq!(index.stealth_count(0, 77), 1); + assert!(index.is_stealthed(0, 77)); + assert!(!index.is_stealthed(1, 78)); + assert_eq!(index.on_target(0, 77).len(), 2); assert_eq!(index.stats(), (3, 2)); // An update lands on the same (shard, id) and replaces the row. let mut refreshed = aura(2, 77, 0xA0); refreshed.amount = 25; index.upsert(0, &refreshed); - assert_eq!(index.on_target(77).len(), 2); + assert_eq!(index.on_target(0, 77).len(), 2); assert_eq!( index - .on_target(77) + .on_target(0, 77) .iter() .find(|a| a.id == 2) .map(|a| a.amount), @@ -599,12 +600,12 @@ fn aura_index_insert_update_delete_by_shard_and_id() { // A delete on shard 1 of id 1 must not touch shard 0's id 1. index.remove(1, &aura(1, 78, 0xA0)); - assert!(index.on_target(78).is_empty()); - assert_eq!(index.stealth_count(77), 1); + assert!(index.on_target(1, 78).is_empty()); + assert_eq!(index.stealth_count(0, 77), 1); index.remove(0, &aura(1, 77, A_STEALTH)); - assert_eq!(index.stealth_count(77), 0); + assert_eq!(index.stealth_count(0, 77), 0); index.remove(0, &aura(1, 77, A_STEALTH)); // idempotent - assert_eq!(index.on_target(77).len(), 1); + assert_eq!(index.on_target(0, 77).len(), 1); index.remove(0, &aura(2, 77, 0xA0)); assert_eq!(index.stats(), (0, 0)); } @@ -614,12 +615,365 @@ fn aura_index_reseed_drops_rows_the_caches_no_longer_hold() { let index = AuraIndex::default(); index.upsert(0, &aura(1, 77, 0xA0)); index.upsert(0, &aura(2, 77, 0xA0)); - index.replace_all([(0, aura(2, 77, 0xA0)), (1, aura(9, 80, A_STEALTH))]); - assert_eq!(index.on_target(77).len(), 1); - assert!(index.is_stealthed(80)); + index.replace_shard(0, [aura(2, 77, 0xA0)]); + index.replace_shard(1, [aura(9, 80, A_STEALTH)]); + assert_eq!(index.on_target(0, 77).len(), 1); + assert!(index.is_stealthed(1, 80)); assert_eq!(index.stats(), (2, 2)); } +#[test] +fn a_transfer_keeps_each_shards_auras_independent() { + let index = AuraIndex::default(); + let mut source = aura(1, PLAYER_BASE, A_STEALTH); + source.amount = 50; + let mut destination = aura(1, PLAYER_BASE, 0xA4); + destination.amount = 25; + index.upsert(0, &source); + index.upsert(1, &destination); + + assert_eq!(index.on_target(1, PLAYER_BASE), vec![destination.clone()]); + assert!(!index.is_stealthed(1, PLAYER_BASE)); + assert!(index.is_stealthed(0, PLAYER_BASE)); + index.replace_shard(0, []); + index.remove(0, &source); + assert_eq!(index.on_target(1, PLAYER_BASE), vec![destination]); +} + +#[test] +fn old_shard_combat_cannot_address_a_transferred_character() { + let mut rng = Rng::new(74); + let realm = spread_realm(&mut rng); + let transferred = &realm.viewers[0]; + let cast = cast(&realm, transferred.self_guid, transferred.self_guid); + let combat = combat(&realm, transferred.self_guid, transferred.self_guid); + realm + .view + .add_viewer_on_shard(transferred.clone(), CellKey::at(1, 0, 4, 4), 1); + realm.view.spatial.upsert_entity( + EntityLayer::WorldEntity, + transferred.self_guid, + CellKey::at(1, 0, 4, 4), + 1, + 0, + ); + + cast_event_appeared(&realm.view, 0, &cast); + combat_event_appeared(&realm.view, 0, &combat); + let mut ranged = melee(transferred.self_guid, 0); + ranged.ranged_spell_id = 75; + melee_disengaged(&realm.view, 0, &ranged); + assert!(realm.queues[0].try_recv().is_err()); + assert!(aura_audience(&realm.view, 0, &aura(1, transferred.self_guid, 0xA0)).is_empty()); + + // A current-shard owner still receives its private stop with no indexed anchor. + realm + .view + .spatial + .remove_entity(EntityLayer::WorldEntity, transferred.self_guid, 1); + melee_disengaged(&realm.view, 1, &ranged); + let Outbound::Job(job) = realm.queues[0].try_recv().unwrap() else { + panic!("expected a Relay job"); + }; + assert!(matches!( + job().as_slice(), + [Outbound::One( + wow_world_messages::vanilla::opcodes::ServerOpcodeMessage::SMSG_CANCEL_AUTO_REPEAT + )] + )); +} + +#[test] +fn spatial_candidates_also_stay_on_the_event_shard() { + let view = WorldView::new(true); + let (tx, rx) = SessionTx::with_depth(0); + let observer = viewer(1, PLAYER_BASE, tx); + let cell = CellKey::at(0, 0, 0, 0); + view.add_viewer_on_shard(observer, cell, 1); + assert!(view + .cell_audience(0, Some(cell), BOX_HALF_SPAN, &[]) + .is_empty()); + assert!(rx.try_recv().is_err()); +} + +#[test] +fn late_source_deletes_preserve_the_destination_entity_and_its_client_object() { + let view = WorldView::new(true); + let (tx, rx) = SessionTx::with_depth(0); + let observer = viewer(1, PLAYER_BASE, tx); + observer.created.lock().unwrap().insert(CREATURE_BASE); + let source = CellKey::at(0, 0, 0, 0); + let destination = CellKey::at(1, 0, 0, 0); + view.add_viewer_on_shard(observer.clone(), destination, 1); + for layer in [EntityLayer::WorldEntity, EntityLayer::GameObject] { + view.spatial + .upsert_entity(layer, CREATURE_BASE, source, 0, 0); + view.spatial + .upsert_entity(layer, CREATURE_BASE, destination, 1, 0); + } + + entity_vanished(&view, 0, CREATURE_BASE, 0); + gameobject_vanished(&view, 0, CREATURE_BASE); + assert!(rx.try_recv().is_err()); + assert!(observer.created.lock().unwrap().contains(&CREATURE_BASE)); + assert!(view + .spatial + .can_see(EntityLayer::WorldEntity, observer.session, CREATURE_BASE)); + assert!(view + .spatial + .can_see(EntityLayer::GameObject, observer.session, CREATURE_BASE)); +} + +#[test] +fn reconnect_removes_missing_rows_through_the_registered_viewers_writer() { + let view = Arc::new(WorldView::new(true)); + let (tx, rx) = SessionTx::with_depth(0); + let arrival = crate::codec::EntityView { + guid: PLAYER_BASE, + ..Default::default() + }; + let registration = super::super::subscriptions::PlayerSubscriptions::registered_for_test( + view.clone(), + PLAYER_BASE, + &arrival, + tx, + ); + let observer = view.viewer_of_owner(OwnerGuid(PLAYER_BASE)).unwrap(); + let cell = CellKey::of_position(0, 0, 0.0, 0.0); + for (layer, guid) in [ + (EntityLayer::WorldEntity, CREATURE_BASE), + (EntityLayer::GameObject, CREATURE_BASE + 1), + ] { + view.spatial.upsert_entity(layer, guid, cell, 0, 0); + if layer == EntityLayer::WorldEntity { + observer.created.lock().unwrap().insert(guid); + } + } + + reconcile_shard(&view, 0, vec![], vec![], vec![]); + let mut destroyed = HashSet::new(); + while let Ok(Outbound::Job(job)) = rx.try_recv() { + for packet in job() { + let Outbound::One( + wow_world_messages::vanilla::opcodes::ServerOpcodeMessage::SMSG_DESTROY_OBJECT( + message, + ), + ) = packet + else { + panic!("expected DESTROY for a missing snapshot row"); + }; + destroyed.insert(message.guid.guid()); + } + } + assert_eq!(destroyed, HashSet::from([CREATURE_BASE, CREATURE_BASE + 1])); + assert_eq!( + *observer.created.lock().unwrap(), + HashSet::from([PLAYER_BASE]) + ); + assert_eq!(view.spatial.viewer_cell(observer.session), Some(cell)); + drop(registration); + assert!(view.viewer_of_owner(OwnerGuid(PLAYER_BASE)).is_none()); + assert_eq!(view.spatial.viewer_cell(observer.session), None); +} + +#[test] +fn a_gameobject_leaving_the_box_uses_its_own_destroy_path() { + let view = WorldView::new(true); + let (tx, rx) = SessionTx::with_depth(0); + let observer = viewer(1, PLAYER_BASE, tx); + view.add_viewer_on_shard(observer, CellKey::at(0, 0, 0, 0), 0); + view.spatial.upsert_entity( + EntityLayer::GameObject, + CREATURE_BASE, + CellKey::at(0, 0, 0, 0), + 0, + 0, + ); + let row = GameObject { + guid: CREATURE_BASE, + template_entry: 1, + map_id: 0, + instance_id: 0, + grid_x: 10, + grid_y: 10, + cell: 0, + x: 0.0, + y: 0.0, + z: 0.0, + orientation: 0.0, + state: 0, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + respawn_at_micros: 0, + rotation_0: 0.0, + rotation_1: 0.0, + rotation_2: 0.0, + rotation_3: 1.0, + }; + gameobject_appeared(&view, 0, &row); + let Outbound::Job(job) = rx.try_recv().unwrap() else { + panic!("expected the game object's Relay job"); + }; + assert!(matches!(job().as_slice(), [Outbound::One( + wow_world_messages::vanilla::opcodes::ServerOpcodeMessage::SMSG_DESTROY_OBJECT(message) + )] if message.guid.guid() == CREATURE_BASE)); +} + +#[test] +fn reconnect_clears_the_action_bar_of_a_pet_missing_from_the_snapshot() { + use wow_world_messages::vanilla::opcodes::ServerOpcodeMessage; + + let view = Arc::new(WorldView::new(true)); + let (tx, rx) = SessionTx::with_depth(0); + let arrival = crate::codec::EntityView { + guid: PLAYER_BASE, + ..Default::default() + }; + let _registration = super::super::subscriptions::PlayerSubscriptions::registered_for_test( + view.clone(), + PLAYER_BASE, + &arrival, + tx, + ); + let observer = view.viewer_of_owner(OwnerGuid(PLAYER_BASE)).unwrap(); + observer.created.lock().unwrap().insert(CREATURE_BASE); + view.spatial.upsert_entity( + EntityLayer::WorldEntity, + CREATURE_BASE, + CellKey::of_position(0, 0, 0.0, 0.0), + 0, + PLAYER_BASE, + ); + + reconcile_shard(&view, 0, vec![], vec![], vec![]); + let Outbound::Job(job) = rx.try_recv().unwrap() else { + panic!("expected the missing pet's Relay job"); + }; + assert!(matches!( + job().as_slice(), + [ + Outbound::One(ServerOpcodeMessage::SMSG_DESTROY_OBJECT(_)), + Outbound::One(ServerOpcodeMessage::SMSG_UPDATE_OBJECT(_)), + Outbound::One(ServerOpcodeMessage::SMSG_PET_SPELLS(_)), + ] + )); + assert_eq!( + *observer.created.lock().unwrap(), + HashSet::from([PLAYER_BASE]) + ); +} + +#[test] +fn an_old_connection_cannot_change_a_new_shard_snapshot() { + let view = Arc::new(WorldView::new(true)); + let mut old = None; + view.with_new_generation(0, |generation| old = Some(generation)); + let old = old.unwrap(); + let cell = CellKey::at(0, 0, 0, 0); + ShardGeneration::run(Some(&old), || { + view.spatial + .upsert_entity(EntityLayer::WorldEntity, CREATURE_BASE, cell, 0, 0); + view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); + }); + view.with_new_generation(0, |_| reconcile_shard(&view, 0, vec![], vec![], vec![])); + + ShardGeneration::run(Some(&old), || { + view.spatial + .upsert_entity(EntityLayer::WorldEntity, CREATURE_BASE, cell, 0, 0); + view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); + }); + assert_eq!( + view.spatial + .entity_cell(EntityLayer::WorldEntity, CREATURE_BASE), + None + ); + assert!(view.auras.on_target(0, CREATURE_BASE).is_empty()); +} + +#[test] +fn a_reconnect_waits_for_an_old_callback_already_in_progress() { + let view = Arc::new(WorldView::new(true)); + let mut generation = None; + view.with_new_generation(0, |current| generation = Some(current)); + let generation = generation.unwrap(); + let (entered, started) = std::sync::mpsc::channel(); + let (release, released) = std::sync::mpsc::channel(); + let (attempt, attempted) = std::sync::mpsc::channel(); + let (complete, completed) = std::sync::mpsc::channel(); + std::thread::scope(|threads| { + let old_view = view.clone(); + threads.spawn(move || { + ShardGeneration::run(Some(&generation), || { + entered.send(()).unwrap(); + released.recv().unwrap(); + old_view.spatial.upsert_entity( + EntityLayer::WorldEntity, + CREATURE_BASE, + CellKey::at(0, 0, 0, 0), + 0, + 0, + ); + old_view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); + }) + }); + started.recv().unwrap(); + threads.spawn(|| { + attempt.send(()).unwrap(); + view.with_new_generation(0, |_| { + reconcile_shard(&view, 0, vec![], vec![], vec![]); + }); + complete.send(()).unwrap(); + }); + attempted.recv().unwrap(); + assert!(completed + .recv_timeout(std::time::Duration::from_millis(20)) + .is_err()); + release.send(()).unwrap(); + }); + completed.recv().unwrap(); + assert_eq!( + view.spatial + .entity_cell(EntityLayer::WorldEntity, CREATURE_BASE), + None + ); + assert!(view.auras.on_target(0, CREATURE_BASE).is_empty()); +} + +#[test] +fn distant_characters_do_not_add_jobs_to_a_local_combat_event() { + let mut rng = Rng::new(810); + let realm = realm(&mut rng, 3, 1, 1.0); + let row = combat(&realm, realm.viewers[0].self_guid, realm.creatures[0]); + combat_event_appeared(&realm.view, 0, &row); + assert_eq!( + realm + .queues + .iter() + .filter(|rx| rx.try_recv().is_ok()) + .count(), + 3 + ); + + let mut distant = Vec::new(); + for i in 0..2000 { + let (tx, rx) = SessionTx::with_depth(0); + let observer = viewer(100 + i, PLAYER_BASE + 100 + i, tx); + realm + .view + .add_viewer_on_shard(observer, CellKey::at(0, 0, 100, 100), 0); + distant.push(rx); + } + combat_event_appeared(&realm.view, 0, &row); + assert_eq!( + realm + .queues + .iter() + .filter(|rx| rx.try_recv().is_ok()) + .count(), + 3 + ); + assert!(distant.iter().all(|rx| rx.try_recv().is_err())); +} + /// The aura audience follows the viewer's anchor: walking into the target's box adds the viewer, /// walking out removes it, with nothing re-registered. #[test] @@ -634,24 +988,26 @@ fn aura_audience_follows_a_viewer_recenter() { stealther, CellKey::at(0, 0, 10, 10), 0, + 0, ); let row = aura(1, stealther, A_STEALTH); - assert!(aura_audience(&view, &row).is_empty()); + assert!(aura_audience(&view, 0, &row).is_empty()); view.spatial .move_viewer_delta(watcher.session, CellKey::at(0, 0, 9, 9)); - assert_eq!(sessions(aura_audience(&view, &row)), HashSet::from([1])); + assert_eq!(sessions(aura_audience(&view, 0, &row)), HashSet::from([1])); view.spatial .move_viewer_delta(watcher.session, CellKey::at(0, 0, 20, 20)); - assert!(aura_audience(&view, &row).is_empty()); + assert!(aura_audience(&view, 0, &row).is_empty()); // The target moves under a resident viewer. view.spatial.upsert_entity( EntityLayer::WorldEntity, stealther, CellKey::at(0, 0, 21, 21), 0, + 0, ); - assert_eq!(sessions(aura_audience(&view, &row)), HashSet::from([1])); + assert_eq!(sessions(aura_audience(&view, 0, &row)), HashSet::from([1])); } /// Every aura read the relays make goes through the index; a whole-cache scan must not creep @@ -675,8 +1031,10 @@ fn aura_relays_never_scan_the_aura_cache() { "{signature} must read the aura index" ); } - let arm = - crate::test_scan::code_of(include_str!("../world_view.rs"), "pub(crate) fn arm_shard"); + let arm = crate::test_scan::code_of( + include_str!("../world_view.rs"), + "fn register_shard_callbacks", + ); assert!( !arm.contains(".game_aura().iter()"), "the pump scans the aura cache" diff --git a/gateway/src/stdb/world_view/relay_bench.rs b/gateway/src/stdb/world_view/relay_bench.rs index 70baaaeb2..1de33e737 100644 --- a/gateway/src/stdb/world_view/relay_bench.rs +++ b/gateway/src/stdb/world_view/relay_bench.rs @@ -1,4 +1,4 @@ -//! Relay scaling harness: a synthetic realm in memory, no sockets, no database. +//! Relay scaling measurement using a synthetic realm in memory, without sockets or a database. //! //! Run with `cargo test -p lyracore-gateway --release relay_scaling -- --ignored --nocapture`. //! It prints one row per population size: the pump cost of a burst of combat, cast and impact @@ -16,6 +16,13 @@ use std::time::{Duration, Instant}; const PLAYERS_PER_CELL: usize = 8; const CREATURES_PER_PLAYER: usize = 4; const AURAS_PER_PLAYER: usize = 5; + +#[derive(Clone, Copy, Debug)] +enum Workload { + Spread, + Dense, + AuraHeavy, +} const AURA_BURST: usize = 50; const PLAYER_GUID_BASE: u64 = 0x0000_0000_0001_0000; const CREATURE_GUID_BASE: u64 = 0xF130_0000_0001_0000; @@ -83,11 +90,13 @@ fn aura(id: u64, target_guid: u64, eff_kind: u8) -> Aura { } } -/// `players` viewers and `4 x players` creatures spread over a square of cells at eight players -/// per cell, every viewer's `created` set seeded with what its box can see, and five auras per -/// player spread over every entity. -fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { - let side = ((players / PLAYERS_PER_CELL) as f64).sqrt().ceil() as i32; +/// Build Characters, four creatures per Character, and the selected cell and aura density. +/// Seed each viewer's created set from its visible cells before dispatching events. +fn build(players: usize, rng: &mut Rng, workload: Workload) -> SyntheticRealm { + let side = match workload { + Workload::Dense => 1, + _ => ((players / PLAYERS_PER_CELL) as f64).sqrt().ceil() as i32, + }; let cell = |rng: &mut Rng| { CellKey::at( 0, @@ -105,7 +114,7 @@ fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { let v = viewer(view.next_session_id(), guid, tx); let anchor = cell(rng); view.spatial - .upsert_entity(EntityLayer::WorldEntity, guid, anchor, 0); + .upsert_entity(EntityLayer::WorldEntity, guid, anchor, 0, 0); view.add_viewer_on_shard(v.clone(), anchor, 0); viewers.push(v); queues.push(rx); @@ -115,7 +124,7 @@ fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { .collect(); for (guid, key) in &creatures { view.spatial - .upsert_entity(EntityLayer::WorldEntity, *guid, *key, 0); + .upsert_entity(EntityLayer::WorldEntity, *guid, *key, 0, 0); } for v in &viewers { let visible = view @@ -127,9 +136,16 @@ fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { .extend(visible.into_iter().map(|(guid, _)| guid)); } let entities = players + creatures.len(); - let auras: Vec = (0..players * AURAS_PER_PLAYER) + let auras_per_character = match workload { + Workload::AuraHeavy => 32, + _ => AURAS_PER_PLAYER, + }; + let auras: Vec = (0..players * auras_per_character) .map(|i| { - let pick = rng.below(entities); + let pick = match workload { + Workload::AuraHeavy => i / auras_per_character, + _ => rng.below(entities), + }; let target = if pick < players { PLAYER_GUID_BASE + pick as u64 } else { @@ -143,8 +159,7 @@ fn build(players: usize, rng: &mut Rng) -> SyntheticRealm { aura(1 + i as u64, target, kind) }) .collect(); - view.auras - .replace_all(auras.iter().map(|row| (0, row.clone()))); + view.auras.replace_shard(0, auras.iter().cloned()); SyntheticRealm { view, viewers, @@ -303,7 +318,7 @@ fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Durati for _ in 0..AURA_BURST { let row = &realm.auras[rng.below(realm.auras.len())]; let started = Instant::now(); - let recipients = aura_audience(&realm.view, row); + let recipients = aura_audience(&realm.view, 0, row); audience += started.elapsed(); candidates += recipients.len(); for viewer in recipients { @@ -314,7 +329,7 @@ fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Durati } let started = Instant::now(); let _ = super::super::subscriptions::aura_sync( - realm.view.auras.on_target(row.target_guid).into_iter(), + realm.view.auras.on_target(0, row.target_guid).into_iter(), row.target_guid, ); sync += started.elapsed(); @@ -329,65 +344,75 @@ fn aura_burst(realm: &SyntheticRealm, rng: &mut Rng) -> (Duration, usize, Durati fn relay_scaling() { println!(); println!( - "| players | creatures | auras | relay | pump us/event | jobs/event | drain us/event | packets/event |" + "| players | creatures | auras | relay | pump us/event | jobs/event | drain us/event | packets/event | jobs total | packets total |" ); - println!("|---|---|---|---|---|---|---|---|"); - for players in [500usize, 1000, 2000] { - let mut rng = Rng::new(0x5CA1_AB1E_0000_0001 + players as u64); - let realm = build(players, &mut rng); - let n = players; - let combat = { - let rows: Vec = (0..n).map(|i| realm.combat_row(i, &mut rng)).collect(); - burst(&realm, |i| combat_event_appeared(&realm.view, &rows[i])) - }; - let cast = { - let rows: Vec = (0..n).map(|i| realm.cast_row(i, &mut rng)).collect(); - burst(&realm, |i| cast_event_appeared(&realm.view, &rows[i])) - }; - let impact = { - let rows: Vec = - (0..n).map(|i| realm.impact_row(i, &mut rng)).collect(); - burst(&realm, |i| impact_appeared(&realm.view, &rows[i])) - }; - for (name, b) in [("combat", combat), ("cast", cast), ("impact", impact)] { + println!("|---|---|---|---|---|---|---|---|---|---|"); + for workload in [Workload::Spread, Workload::Dense, Workload::AuraHeavy] { + println!("workload={workload:?}; spread holds density near eight Characters/cell; dense uses one cell; aura-heavy gives each Character 32 auras"); + for players in [500usize, 1000, 2000] { + let mut rng = Rng::new(0x5CA1_AB1E_0000_0001 + players as u64); + let realm = build(players, &mut rng, workload); + let n = players; + let combat = { + let rows: Vec = + (0..n).map(|i| realm.combat_row(i, &mut rng)).collect(); + burst(&realm, |i| combat_event_appeared(&realm.view, 0, &rows[i])) + }; + let cast = { + let rows: Vec = + (0..n).map(|i| realm.cast_row(i, &mut rng)).collect(); + burst(&realm, |i| cast_event_appeared(&realm.view, 0, &rows[i])) + }; + let impact = { + let rows: Vec = + (0..n).map(|i| realm.impact_row(i, &mut rng)).collect(); + burst(&realm, |i| impact_appeared(&realm.view, 0, &rows[i])) + }; + for (name, b) in [("combat", combat), ("cast", cast), ("impact", impact)] { + println!( + "| {players} | {} | {} | {name} | {:.1} | {:.0} | {:.1} | {:.1} | {} | {} |", + realm.creatures.len(), + realm.auras.len(), + micros_per(b.pump, n), + b.jobs as f64 / n as f64, + micros_per(b.drain, n), + b.packets as f64 / n as f64, + b.jobs, + b.packets, + ); + } + let (audience, candidates, sync, synced) = aura_burst(&realm, &mut rng); println!( - "| {players} | {} | {} | {name} | {:.1} | {:.0} | {:.1} | {:.1} |", + "| {players} | {} | {} | aura | {:.1} | {:.0} | {:.1} | {:.1} | {} | {} |", realm.creatures.len(), realm.auras.len(), - micros_per(b.pump, n), - b.jobs as f64 / n as f64, - micros_per(b.drain, n), - b.packets as f64 / n as f64, + micros_per(audience, AURA_BURST), + candidates as f64 / AURA_BURST as f64, + micros_per(sync, AURA_BURST), + synced as f64 / AURA_BURST as f64, + candidates, + synced, + ); + let yells: Vec = (0..n).map(|i| realm.yell_row(i)).collect(); + let started = Instant::now(); + let candidates: usize = yells + .iter() + .map(|row| chat_audience(&realm.view, 0, row).len()) + .sum(); + let audience = started.elapsed(); + println!( + "| {players} | {} | {} | yell | {:.1} | {:.0} | n/a | n/a | {} | n/a |", + realm.creatures.len(), + realm.auras.len(), + micros_per(audience, n), + candidates as f64 / n as f64, + candidates, ); } - let (audience, candidates, sync, synced) = aura_burst(&realm, &mut rng); - println!( - "| {players} | {} | {} | aura | {:.1} | {:.0} | {:.1} | {:.1} |", - realm.creatures.len(), - realm.auras.len(), - micros_per(audience, AURA_BURST), - candidates as f64 / AURA_BURST as f64, - micros_per(sync, AURA_BURST), - synced as f64 / AURA_BURST as f64, - ); - let yells: Vec = (0..n).map(|i| realm.yell_row(i)).collect(); - let started = Instant::now(); - let candidates: usize = yells - .iter() - .map(|row| chat_audience(&realm.view, row).len()) - .sum(); - let audience = started.elapsed(); - println!( - "| {players} | {} | {} | yell | {:.1} | {:.0} | n/a | n/a |", - realm.creatures.len(), - realm.auras.len(), - micros_per(audience, n), - candidates as f64 / n as f64, - ); } println!(); println!("aura columns: pump = audience selection, jobs = candidate viewers, drain = array syncs built, packets = viewers past the visibility gate"); println!( - "yell columns: pump = audience selection over the widened span, jobs = candidate viewers" + "yell columns: pump = audience selection over the widened span, jobs = candidate viewers; timings exclude sockets, concurrent writers, and Module transactions" ); } From 19acfb66569c8886d15f816ec4f73f93d7999d6c Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Fri, 4 Sep 2026 23:22:26 +0200 Subject: [PATCH 4/9] fix(gateway): scope motion and recovery relays to their shard --- gateway/src/stdb/connection.rs | 64 +++-- gateway/src/stdb/subscriptions.rs | 74 ++++- gateway/src/stdb/world_index.rs | 2 + gateway/src/stdb/world_view.rs | 270 ++++++++++-------- .../stdb/world_view/cell_audience_tests.rs | 110 +++++++ 5 files changed, 371 insertions(+), 149 deletions(-) diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index 425456c7a..4b430e110 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -1503,6 +1503,7 @@ mod pump_tests { row: Mutex>, projected: Mutex>, registered: AtomicBool, + pending_registration: AtomicBool, } #[tokio::test] @@ -1526,35 +1527,42 @@ mod pump_tests { #[tokio::test] async fn a_delta_arriving_during_snapshot_setup_is_applied_after_the_snapshot() { - let cache = ExternalCache::default(); - *cache.row.lock().unwrap() = Some(1); - let (setup, work) = mpsc::unbounded_channel::>(); - let (incoming, messages) = mpsc::unbounded_channel(); - let messages = tokio::sync::Mutex::new(messages); - setup - .send(Box::new(move |cache| { - cache.registered.store(true, Ordering::Relaxed); - let snapshot = *cache.row.lock().unwrap(); - incoming.send(Some(2)).unwrap(); - incoming.send(None).unwrap(); - *cache.projected.lock().unwrap() = snapshot; - })) - .unwrap(); - - let result = pump_messages(&cache, work, || async { - let Some(Some(row)) = messages.lock().await.recv().await else { - return Err(()); - }; - *cache.row.lock().unwrap() = Some(row); - if cache.registered.load(Ordering::Relaxed) { - *cache.projected.lock().unwrap() = Some(row); - } - Ok(()) - }) - .await; + for (snapshot, delta) in [(None, Some(2)), (Some(1), Some(2)), (Some(1), None)] { + let cache = ExternalCache::default(); + *cache.row.lock().unwrap() = snapshot; + let (setup, work) = mpsc::unbounded_channel::>(); + let (incoming, messages) = mpsc::unbounded_channel::, ()>>(); + let messages = tokio::sync::Mutex::new(messages); + setup + .send(Box::new(move |cache| { + cache.pending_registration.store(true, Ordering::Relaxed); + let snapshot = *cache.row.lock().unwrap(); + incoming.send(Ok(delta)).unwrap(); + incoming.send(Err(())).unwrap(); + *cache.projected.lock().unwrap() = snapshot; + })) + .unwrap(); + + let result = pump_messages(&cache, work, || async { + // SDK 2.7.1 applies queued registrations before any waiting remote delta. + if cache.pending_registration.swap(false, Ordering::Relaxed) { + cache.registered.store(true, Ordering::Relaxed); + return Ok(()); + } + let Some(Ok(row)) = messages.lock().await.recv().await else { + return Err(()); + }; + *cache.row.lock().unwrap() = row; + if cache.registered.load(Ordering::Relaxed) { + *cache.projected.lock().unwrap() = row; + } + Ok(()) + }) + .await; - assert_eq!(result, Err(())); - assert_eq!(*cache.projected.lock().unwrap(), Some(2)); + assert_eq!(result, Err(())); + assert_eq!(*cache.projected.lock().unwrap(), delta); + } } } diff --git a/gateway/src/stdb/subscriptions.rs b/gateway/src/stdb/subscriptions.rs index d6b065c0c..d71d3d519 100644 --- a/gateway/src/stdb/subscriptions.rs +++ b/gateway/src/stdb/subscriptions.rs @@ -1753,6 +1753,71 @@ pub(crate) fn aura_delete_outbound( out } +/// Reconnect sends one current aura array per target and observer. Owner timers remain per aura, +/// while speed, armor, and sheet values refresh once even when several auras changed offline. +pub(crate) fn aura_snapshot_outbound( + coord: &Coordinator, + view: &WorldView, + shard: super::world_index::ShardId, + viewer: &Viewer, + target_guid: u64, + previous: &[Aura], +) -> Vec { + let current = view.auras.on_target(shard, target_guid); + let stealth_count = current + .iter() + .filter(|row| row.eff_kind == A_STEALTH) + .count(); + let mut out = Vec::new(); + if let Some(stealth) = current + .iter() + .chain(previous) + .find(|row| row.eff_kind == A_STEALTH) + { + out.extend(stealth_visibility( + stealth_count, + view, + viewer.session, + coord, + &viewer.created, + stealth, + viewer.self_guid, + stealth_count > 0, + )); + } + if target_guid != viewer.self_guid && !viewer.created.lock().unwrap().contains(&target_guid) { + return out; + } + out.push(aura_sync(current.iter().cloned(), target_guid)); + if target_guid != viewer.self_guid { + return out; + } + out.extend( + current + .iter() + .filter_map(|row| aura_duration_packet(row, viewer.self_guid)), + ); + out.extend( + current + .iter() + .chain(previous) + .find_map(|row| run_speed_packet(current.iter().cloned(), row, viewer.self_guid)), + ); + out.extend( + current + .iter() + .chain(previous) + .find_map(|row| armor_packet(coord, row, viewer.self_guid)), + ); + out.extend( + current + .iter() + .chain(previous) + .find_map(|row| sheet_packet(coord, row, viewer.self_guid)), + ); + out +} + /// Trade-status relay (#120): `game_trade_event.kind` → the `SMSG_TRADE_STATUS` variant, to the /// row's recipient and nobody else (audience resolved by the caller, the `whisper_event_outbound` /// shape). The kind byte is `lyracore_shared::trade::event_kind` — NOT the vanilla discriminant; @@ -5473,12 +5538,13 @@ mod tests { #[test] fn the_realm_private_relays_ride_the_recipient_keyed_dispatchers() { let body = decommented(top_level_fn_body_of("world_view.rs", "arm_realm_private")); + let compact: String = body.chars().filter(|c| !c.is_whitespace()).collect(); assert!( - body.contains("wire_insert_live(db.game_whisper_event(), \"realm.game_whisper_event.insert\", &view, |v, row| { whisper_appeared(v, row) });"), + compact.contains("wire_insert_live(db.game_whisper_event(),\"realm.game_whisper_event.insert\",&view,None,|v,row|whisper_appeared(v,row),);"), "arm_realm_private no longer relays realm-core whispers through `whisper_appeared`" ); assert!( - body.contains("wire_insert_live(db.game_group_event(), \"realm.game_group_event.insert\", &view, move |v, row| { group_event_appeared(v, &coord, row) });"), + compact.contains("wire_insert_live(db.game_group_event(),\"realm.game_group_event.insert\",&view,None,move|v,row|group_event_appeared(v,&coord,row),);"), "arm_realm_private no longer relays realm-core group events through \ `group_event_appeared` (which also carries the QUEST_SHARE detail JOIN through a \ WORLD handle — realm-core's cache has no quest catalogue)" @@ -6204,7 +6270,7 @@ mod tests { /// freeze after their first step, or never move at all. Same for the creature-leg twin. #[test] fn both_halves_of_the_motion_and_spline_relays_are_registered_286() { - let arm = decommented(top_level_fn_body_of("world_view.rs", "register_shard_callbacks")); + let arm: String = decommented(top_level_fn_body_of("world_view.rs", "register_shard_callbacks")).chars().filter(|c| !c.is_whitespace()).collect(); // #490 factored every registration through `wire_insert`/`wire_update` (world_view.rs), // so the literal `.on_insert(`/`.on_update(` chain off the table handle is gone from // `arm_shard`'s own body — what's left to scan for is the (helper, table, label) triple @@ -6224,7 +6290,7 @@ mod tests { // And the dispatch routes them through the cell index rather than broadcasting. let m = decommented(top_level_fn_body_of("world_view.rs", "motion")); assert!( - m.contains("view.spatial.viewers_of(EntityLayer::WorldEntity, key)"), + m.contains("view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[])"), "peer motion no longer asks the cell index who can see the mover — either every session \ gets every mover (the fan-out this issue removed) or none do" ); diff --git a/gateway/src/stdb/world_index.rs b/gateway/src/stdb/world_index.rs index fafbbbf7b..aa70978db 100644 --- a/gateway/src/stdb/world_index.rs +++ b/gateway/src/stdb/world_index.rs @@ -298,6 +298,7 @@ impl WorldIndex { /// Where the index believes `guid` is: the anchor of a relay driven by a table with no cell of /// its own (an aura names its target, a chat line its speaker). `None` for a guid it has never /// seen (a row on a table it does not index, or one deleted already). + #[cfg(test)] pub fn entity_cell(&self, layer: EntityLayer, guid: u64) -> Option { let inner = self.lock(); Self::layer(&inner, layer) @@ -555,6 +556,7 @@ impl WorldIndex { } /// Live sizes, for the periodic ops log line: `(entities, viewers, occupied entity cells)`. + #[cfg(test)] pub fn stats(&self, layer: EntityLayer) -> (usize, usize, usize) { let inner = self.lock(); let entities = Self::layer(&inner, layer); diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index 1cdb2e656..aff7bdb39 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -230,23 +230,25 @@ impl AuraIndex { } } - /// Called on the shard's pump between messages, after its callbacks are registered. + /// Replace the Shard snapshot and return previous auras grouped by every old or current target. + /// A newly present target has an empty previous set. Each target needs one snapshot Relay. pub(crate) fn replace_shard( &self, shard: ShardId, rows: impl IntoIterator, - ) -> Vec { + ) -> HashMap> { let mut by_target = self.lock(); let targets: Vec<_> = by_target .keys() .filter(|(owner, _)| *owner == shard) .copied() .collect(); - let old = targets + let mut old: HashMap<_, _> = targets .into_iter() - .flat_map(|target| by_target.remove(&target).unwrap_or_default()) + .map(|target| (target.1, by_target.remove(&target).unwrap_or_default())) .collect(); for row in rows { + old.entry(row.target_guid).or_default(); by_target .entry((shard, row.target_guid)) .or_default() @@ -353,6 +355,18 @@ impl WorldView { (registered.shard == shard).then(|| registered.viewer.clone()) } + fn viewer_of_identity_on_shard( + &self, + shard: ShardId, + identity: BoundIdentity, + ) -> Option> { + let registry = self.viewers.read().unwrap(); + let session = registry.session_of_identity.get(&identity.0)?; + let registered = registry.by_session.get(session)?; + (registered.shard == shard).then(|| registered.viewer.clone()) + } + + #[cfg(test)] pub(crate) fn viewer_of_identity(&self, identity: BoundIdentity) -> Option> { let registry = self.viewers.read().unwrap(); let session = registry.session_of_identity.get(&identity.0)?; @@ -566,21 +580,21 @@ fn register_shard_callbacks( "game_addon_message.insert", &view, Some(&generation), - |v, row| addon_message_appeared(v, row), + move |v, row| addon_message_appeared(v, shard, row), ); wire_insert_live( db.game_xp_event(), "game_xp_event.insert", &view, Some(&generation), - |v, row| xp_appeared(v, row), + move |v, row| xp_appeared(v, shard, row), ); wire_insert_live( db.game_levelup_event(), "game_levelup_event.insert", &view, Some(&generation), - |v, row| levelup_appeared(v, row), + move |v, row| levelup_appeared(v, shard, row), ); let quest_insert_coord = coord.clone(); wire_insert( @@ -765,14 +779,14 @@ fn register_shard_callbacks( "game_entity_motion.insert", &view, Some(&generation), - |v, row| motion(v, row), + move |v, row| motion(v, shard, row), ); wire_update( db.game_entity_motion(), "game_entity_motion.update", &view, Some(&generation), - |v, _old, row| motion(v, row), + move |v, _old, row| motion(v, shard, row), ); // ---- game_creature_spline (creature legs) ---------------------------------------------- @@ -781,14 +795,14 @@ fn register_shard_callbacks( "game_creature_spline.insert", &view, Some(&generation), - |v, row| creature_leg(v, row), + move |v, row| creature_leg(v, shard, row), ); wire_update( db.game_creature_spline(), "game_creature_spline.update", &view, Some(&generation), - |v, _old, row| creature_leg(v, row), + move |v, _old, row| creature_leg(v, shard, row), ); // ---- game_taxi_passenger_spline (server-owned player routes) --------------------------- @@ -797,14 +811,14 @@ fn register_shard_callbacks( "game_taxi_passenger_spline.insert", &view, Some(&generation), - |v, row| taxi_spline(v, row), + move |v, row| taxi_spline(v, shard, row), ); wire_update( db.game_taxi_passenger_spline(), "game_taxi_passenger_spline.update", &view, Some(&generation), - |v, _old, row| taxi_spline(v, row), + move |v, _old, row| taxi_spline(v, shard, row), ); // ---- game_roll_event ------------------------------------------------------------------- @@ -1111,14 +1125,14 @@ fn register_shard_callbacks( "game_player_skill.insert", &view, Some(&generation), - |v, row| skill_changed(v, row), + move |v, row| skill_changed(v, shard, row), ); wire_update( db.game_player_skill(), "game_player_skill.update", &view, Some(&generation), - |v, _old, row| skill_changed(v, row), + move |v, _old, row| skill_changed(v, shard, row), ); } @@ -1315,9 +1329,9 @@ fn entity_changed(view: &Arc, shard: ShardId, old: &WorldEntity, new: if let Some(old_key) = previous { if old_key != key { let still: HashSet = new_recipients.iter().copied().collect(); - for session in view.spatial.viewers_of(EntityLayer::WorldEntity, old_key) { - if !still.contains(&session) { - destroy_job(view, session, new.guid, new.owner_guid); + for viewer in view.cell_audience(shard, Some(old_key), BOX_HALF_SPAN, &[]) { + if !still.contains(&viewer.session) { + destroy_job(view, viewer.session, new.guid, new.owner_guid); } } } @@ -1478,17 +1492,14 @@ fn gameobject_vanished(view: &WorldView, shard: ShardId, guid: u64) { // Motion + creature legs — no index of their own: the row carries its own cell. // =============================================================================================== -fn motion(view: &WorldView, row: &EntityMotion) { +fn motion(view: &WorldView, shard: ShardId, row: &EntityMotion) { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); let row = Arc::new(row.clone()); // The MOVER is deliberately not a recipient (`world_entity_recipients`' self leg is skipped // here): a // player must never receive their own movement echoed back, or they fight the server for // authority over their position. `relay_entity_motion` re-checks it anyway. - for session in view.spatial.viewers_of(EntityLayer::WorldEntity, key) { - let Some(viewer) = view.viewer(session) else { - continue; - }; + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { if viewer.self_guid == row.guid { continue; } @@ -1505,13 +1516,10 @@ fn motion(view: &WorldView, row: &EntityMotion) { } } -fn creature_leg(view: &WorldView, row: &CreatureSpline) { +fn creature_leg(view: &WorldView, shard: ShardId, row: &CreatureSpline) { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); let row = Arc::new(row.clone()); - for session in view.spatial.viewers_of(EntityLayer::WorldEntity, key) { - let Some(viewer) = view.viewer(session) else { - continue; - }; + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { // Same supersession as player motion — latest leg per creature wins. viewer .motion_pending @@ -1625,26 +1633,14 @@ fn dynobj_vanished(view: &WorldView, shard: ShardId, row: &DynamicObject) { /// A passenger spline is always sent to its owner. Nearby sessions receive it only after the /// outbound visibility check confirms that the passenger already exists in their client world. -fn taxi_spline(view: &WorldView, row: &TaxiPassengerSpline) { +fn taxi_spline(view: &WorldView, shard: ShardId, row: &TaxiPassengerSpline) { let key = CellKey::at(row.map_id, row.instance_id, row.grid_x, row.grid_y); - let mut sessions: std::collections::HashSet<_> = - view.spatial.viewers_of(EntityLayer::WorldEntity, key).into_iter().collect(); - if let Some(owner) = view.session_of_owner(row.character_guid) { - sessions.insert(owner); - } let row = row.clone(); - for session in sessions { - let Some(viewer) = view.viewer(session) else { - continue; - }; + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[row.character_guid]) { let row = row.clone(); let tx = viewer.tx.clone(); enqueue(&tx, move || { - super::subscriptions::taxi_spline_outbound( - &viewer.created, - viewer.self_guid, - &row, - ) + super::subscriptions::taxi_spline_outbound(&viewer.created, viewer.self_guid, &row) }); } } @@ -1663,8 +1659,10 @@ fn teleport_appeared(view: &WorldView, shard: ShardId, row: &TeleportEvent) { } /// Addon messages are identity-addressed, with the addressed viewer as the wire sender. -fn addon_message_appeared(view: &WorldView, row: &AddonMessage) { - let Some(viewer) = view.viewer_of_identity(BoundIdentity(row.recipient_identity)) else { +fn addon_message_appeared(view: &WorldView, shard: ShardId, row: &AddonMessage) { + let Some(viewer) = + view.viewer_of_identity_on_shard(shard, BoundIdentity(row.recipient_identity)) + else { return; }; let row = row.clone(); @@ -1675,8 +1673,10 @@ fn addon_message_appeared(view: &WorldView, row: &AddonMessage) { } /// XP rows are delivered to the one viewer bound to their recipient identity. -fn xp_appeared(view: &WorldView, row: &XpEvent) { - let Some(viewer) = view.viewer_of_identity(BoundIdentity(row.recipient_identity)) else { +fn xp_appeared(view: &WorldView, shard: ShardId, row: &XpEvent) { + let Some(viewer) = + view.viewer_of_identity_on_shard(shard, BoundIdentity(row.recipient_identity)) + else { return; }; let row = row.clone(); @@ -1685,8 +1685,10 @@ fn xp_appeared(view: &WorldView, row: &XpEvent) { } /// Level-up rows are delivered to the one viewer bound to their recipient identity. -fn levelup_appeared(view: &WorldView, row: &LevelupEvent) { - let Some(viewer) = view.viewer_of_identity(BoundIdentity(row.recipient_identity)) else { +fn levelup_appeared(view: &WorldView, shard: ShardId, row: &LevelupEvent) { + let Some(viewer) = + view.viewer_of_identity_on_shard(shard, BoundIdentity(row.recipient_identity)) + else { return; }; let row = row.clone(); @@ -1934,11 +1936,8 @@ fn breath_relay_appeared(view: &WorldView, row: &BreathRelayEvent) { /// A skill row changed (line learned / skill-up). Self-only family, same owner-session-lookup /// audience as [`rest_state_appeared`] — the slot allocation runs in the job on the owner's own /// writer thread, against the viewer's `skill_slots` (the same map the per-player leg uses). -fn skill_changed(view: &WorldView, row: &PlayerSkill) { - let Some(session) = view.session_of_owner(row.character_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +fn skill_changed(view: &WorldView, shard: ShardId, row: &PlayerSkill) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; let (line, cur, max) = (row.skill_line, row.current, row.max_rank); @@ -2899,7 +2898,7 @@ mod family_audience_tests { // Removing one table's row cannot erase the equal-valued GUID in the other table. assert_eq!( view.spatial.remove_entity(EntityLayer::WorldEntity, 77, 0), - Some(entering) + Some((entering, 0)) ); assert!(view .spatial @@ -3125,62 +3124,81 @@ mod family_audience_tests { view.add_viewer_on_shard(owner.clone(), anchor, 0); view.add_viewer_on_shard(other, anchor, 0); - xp_appeared( - &view, - &XpEvent { - id: 1, - recipient_identity: owner.bound_identity, - killed_guid: 77, - total_exp: 123, - created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, - is_kill: true, - }, - ); - levelup_appeared( - &view, - &LevelupEvent { - id: 2, - recipient_identity: owner.bound_identity, - new_level: 10, - health_gained: 20, - created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, - mana_gained: 15, - strength_gained: 1, - agility_gained: 2, - stamina_gained: 3, - intellect_gained: 4, - spirit_gained: 5, - }, - ); - addon_message_appeared( - &view, - &AddonMessage { - id: 3, - recipient_identity: owner.bound_identity, - cmd: "progress".to_string(), - payload: "ready".to_string(), - created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, - }, - ); + for shard in [1, 0] { + xp_appeared( + &view, + shard, + &XpEvent { + id: 1, + recipient_identity: owner.bound_identity, + killed_guid: 77, + total_exp: 123, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + is_kill: true, + }, + ); + levelup_appeared( + &view, + shard, + &LevelupEvent { + id: 2, + recipient_identity: owner.bound_identity, + new_level: 10, + health_gained: 20, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + mana_gained: 15, + strength_gained: 1, + agility_gained: 2, + stamina_gained: 3, + intellect_gained: 4, + spirit_gained: 5, + }, + ); + addon_message_appeared( + &view, + shard, + &AddonMessage { + id: 3, + recipient_identity: owner.bound_identity, + cmd: "progress".to_string(), + payload: "ready".to_string(), + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + }, + ); - assert!(other_rx.try_recv().is_err(), "an unrelated identity receives nothing"); - assert!(matches!( - queued_job(&owner_rx).as_slice(), - [Outbound::One(ServerOpcodeMessage::SMSG_LOG_XPGAIN(_))] - )); - assert!(matches!( - queued_job(&owner_rx).as_slice(), - [Outbound::One(ServerOpcodeMessage::SMSG_LEVELUP_INFO(_))] - )); - match queued_job(&owner_rx).as_slice() { - [Outbound::Raw { body, .. }] => assert_eq!( - &body[5..13], - &owner.self_guid.to_le_bytes(), - "the addon sender is the addressed viewer" - ), - _ => panic!("addon event must produce one raw SMSG_MESSAGECHAT"), + if shard == 1 { + assert!( + owner_rx.try_recv().is_err(), + "source Shard events cannot reach a destination owner" + ); + assert!(other_rx.try_recv().is_err()); + continue; + } + assert!( + other_rx.try_recv().is_err(), + "an unrelated identity receives nothing" + ); + assert!(matches!( + queued_job(&owner_rx).as_slice(), + [Outbound::One(ServerOpcodeMessage::SMSG_LOG_XPGAIN(_))] + )); + assert!(matches!( + queued_job(&owner_rx).as_slice(), + [Outbound::One(ServerOpcodeMessage::SMSG_LEVELUP_INFO(_))] + )); + match queued_job(&owner_rx).as_slice() { + [Outbound::Raw { body, .. }] => assert_eq!( + &body[5..13], + &owner.self_guid.to_le_bytes(), + "the addon sender is the addressed viewer" + ), + _ => panic!("addon event must produce one raw SMSG_MESSAGECHAT"), + } + assert!( + owner_rx.try_recv().is_err(), + "each identity row queues exactly once" + ); } - assert!(owner_rx.try_recv().is_err(), "each identity row queues exactly once"); } #[test] @@ -3669,7 +3687,7 @@ fn reconcile_shard( objects: Vec, auras: Vec, ) { - let old_auras = view.auras.replace_shard(shard, auras.iter().cloned()); + let aura_targets = view.auras.replace_shard(shard, auras); let removed_entities = view.spatial.replace_shard( EntityLayer::WorldEntity, shard, @@ -3710,13 +3728,25 @@ fn reconcile_shard( } } if let Some(coord) = view.shard(shard) { - for row in old_auras { - let count = view.auras.stealth_count(shard, row.target_guid); - aura_removed(view, &coord, shard, &row, count); - } - for row in auras { - let count = view.auras.stealth_count(shard, row.target_guid); - aura_applied(view, &coord, shard, &row, count); + for (target_guid, previous) in aura_targets { + let previous = Arc::new(previous); + let key = + view.spatial + .entity_cell_on_shard(EntityLayer::WorldEntity, target_guid, shard); + for viewer in view.cell_audience(shard, key, BOX_HALF_SPAN, &[target_guid]) { + let (coord, view, previous) = (coord.clone(), view.clone(), previous.clone()); + let tx = viewer.tx.clone(); + enqueue(&tx, move || { + super::subscriptions::aura_snapshot_outbound( + &coord, + &view, + shard, + &viewer, + target_guid, + &previous, + ) + }); + } } } // Existing viewers keep their CREATE dedup state. Newly resident rows are offered normally. @@ -3726,6 +3756,12 @@ fn reconcile_shard( } let row = Arc::new(row); for session in view.world_entity_recipients(shard, row.guid, entity_key(&row)) { + if view + .viewer(session) + .is_some_and(|v| v.created.lock().unwrap().contains(&row.guid)) + { + continue; + } offer_create_job(view, shard, session, &row); } } diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index 8f6ab294d..ed4f2cbc1 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -640,6 +640,116 @@ fn a_transfer_keeps_each_shards_auras_independent() { assert_eq!(index.on_target(1, PLAYER_BASE), vec![destination]); } +#[test] +fn a_reconnect_groups_aura_refreshes_by_target() { + let index = AuraIndex::default(); + for id in 0..32 { + index.upsert(0, &aura(id, PLAYER_BASE, 0xA0)); + } + index.upsert(0, &aura(32, PLAYER_BASE + 1, A_STEALTH)); + let mut current: Vec<_> = (0..32).map(|id| aura(id, PLAYER_BASE, 0xA0)).collect(); + current.push(aura(33, PLAYER_BASE + 2, 0xA0)); + + let refreshes = index.replace_shard(0, current); + assert_eq!(refreshes.len(), 3, "32 buffs need one target refresh"); + assert_eq!(refreshes[&PLAYER_BASE].len(), 32); + assert_eq!(refreshes[&(PLAYER_BASE + 1)].len(), 1); + assert!(refreshes[&(PLAYER_BASE + 2)].is_empty()); + assert!(index.on_target(0, PLAYER_BASE + 1).is_empty()); +} + +#[test] +fn an_old_shard_cannot_move_or_change_skills_on_a_destination_viewer() { + let view = WorldView::new(true); + let (tx, rx) = SessionTx::with_depth(0); + let destination = viewer(1, PLAYER_BASE, tx); + destination.created.lock().unwrap().insert(CREATURE_BASE); + view.add_viewer_on_shard(destination.clone(), CellKey::at(0, 0, 0, 0), 1); + motion( + &view, + 0, + &EntityMotion { + guid: CREATURE_BASE, + map_id: 0, + instance_id: 0, + grid_x: 0, + grid_y: 0, + opcode: 0, + movement_info: vec![], + seq: 1, + cell: 0, + }, + ); + creature_leg( + &view, + 0, + &CreatureSpline { + guid: CREATURE_BASE, + start_micros: 0, + dur_ms: 100, + sx: 0.0, + sy: 0.0, + sz: 0.0, + dx: 1.0, + dy: 0.0, + dz: 0.0, + map_id: 0, + instance_id: 0, + grid_x: 0, + grid_y: 0, + spline_id: 1, + run: true, + cell: 0, + facing: false, + facing_angle: 0.0, + }, + ); + skill_changed( + &view, + 0, + &PlayerSkill { + id: 1, + character_guid: PLAYER_BASE, + owner_identity: destination.bound_identity, + skill_line: 43, + current: 10, + max_rank: 50, + }, + ); + let spline = TaxiPassengerSpline { + character_guid: PLAYER_BASE, + map_id: 0, + instance_id: 0, + grid_x: 0, + grid_y: 0, + cell: 0, + start_x: 0.0, + start_y: 0.0, + start_z: 0.0, + points: vec![1.0, 0.0, 0.0], + duration_ms: 100, + spline_id: 1, + }; + taxi_spline(&view, 0, &spline); + assert!(rx.try_recv().is_err()); + assert!(destination.motion_pending.entity.lock().unwrap().is_empty()); + assert!(destination + .motion_pending + .creature + .lock() + .unwrap() + .is_empty()); + taxi_spline(&view, 1, &spline); + let Outbound::Job(job) = rx.try_recv().unwrap() else { + panic!("expected the current Shard's taxi Relay"); + }; + assert_eq!( + job().len(), + 1, + "the passenger still receives its own current spline" + ); +} + #[test] fn old_shard_combat_cannot_address_a_transferred_character() { let mut rng = Rng::new(74); From f6212cb7e8a23d13c9b72bc9ba8bde15914df4f1 Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Fri, 4 Sep 2026 23:30:46 +0200 Subject: [PATCH 5/9] fix(gateway): discard relays from departed world sessions --- gateway/src/stdb/subscriptions.rs | 29 ++- gateway/src/stdb/world_view.rs | 239 ++++++++---------- .../stdb/world_view/cell_audience_tests.rs | 65 ++++- gateway/src/stdb/world_view/relay_bench.rs | 1 + 4 files changed, 192 insertions(+), 142 deletions(-) diff --git a/gateway/src/stdb/subscriptions.rs b/gateway/src/stdb/subscriptions.rs index d71d3d519..6fd38f0c6 100644 --- a/gateway/src/stdb/subscriptions.rs +++ b/gateway/src/stdb/subscriptions.rs @@ -64,6 +64,7 @@ impl PlayerSubscriptions { let mut identity = [0; 32]; identity[..8].copy_from_slice(&self_guid.to_le_bytes()); let viewer = Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session: view.next_session_id(), self_guid, bound_identity: spacetimedb_sdk::Identity::from_byte_array(identity), @@ -95,6 +96,9 @@ impl PlayerSubscriptions { /// remain idempotent. pub(crate) fn unregister_viewer(&mut self) { if let (Some(view), Some(viewer)) = (self.view.take(), self.viewer.take()) { + viewer + .active + .store(false, std::sync::atomic::Ordering::Release); view.remove_viewer(viewer.session); } } @@ -1114,15 +1118,10 @@ pub(crate) fn relay_breath_event( /// viewer's `created` set (no point animating an invisible attacker's swing — the victim's health /// still moves via the entity VALUES relay if the victim is in scope). /// -/// `tx` is used ONLY for the delayed ranged-impact damage log (auto-shot: the number arrives WITH the -/// arrow, via a thread per landed shot — a shared timer wheel if archer armies happen); every -/// immediate packet is RETURNED so the shared path writes it at the job's queue position. -pub(crate) fn combat_event_outbound( - tx: &SessionTx, - created: &Mutex>, - row: &CombatEvent, -) -> Vec { - if !created.lock().unwrap().contains(&row.attacker_guid) { +/// Immediate packets return to the writer at this job's queue position. A delayed damage +/// log returns through the same World Session lifetime check after its delay. +pub(crate) fn combat_event_outbound(viewer: &Arc, row: &CombatEvent) -> Vec { + if !viewer.created.lock().unwrap().contains(&row.attacker_guid) { return Vec::new(); } let mut out = Vec::new(); @@ -1170,11 +1169,11 @@ pub(crate) fn combat_event_outbound( // the health there) — hold the LOG to the same moment so the number arrives // WITH the arrow, not at the muzzle. if row.impact_delay_ms > 0 { - let tx_late = tx.clone(); + let viewer = viewer.clone(); let delay = std::time::Duration::from_millis(row.impact_delay_ms as u64); std::thread::spawn(move || { std::thread::sleep(delay); - let _ = tx_late.send(msg); + super::world_view::enqueue(viewer, move |_| vec![msg]); }); } else { out.push(msg); @@ -1192,9 +1191,9 @@ pub(crate) fn combat_event_outbound( row.blocked_amount, 0, ); - out.push(Outbound::One(ServerOpcodeMessage::SMSG_ATTACKERSTATEUPDATE( - Box::new(m), - ))); + out.push(Outbound::One( + ServerOpcodeMessage::SMSG_ATTACKERSTATEUPDATE(Box::new(m)), + )); } // C2: on a MELEE killing blow, tell the attacker to leave combat stance. The target itself // vanishes via the game_world_entity on_delete → SMSG_DESTROY_OBJECT relay. A RANGED kill @@ -3360,6 +3359,7 @@ impl Coordinator { .store(is_ghost, std::sync::atomic::Ordering::Relaxed); } let viewer = Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session, self_guid, bound_identity: self_identity, @@ -3668,6 +3668,7 @@ mod tests { fn viewer_with_created(creature_guid: u64) -> Viewer { let (tx, _rx) = SessionTx::with_depth(0); Viewer { + active: std::sync::atomic::AtomicBool::new(true), session: 1, self_guid: 7, bound_identity: spacetimedb_sdk::Identity::from_byte_array([0; 32]), diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index aff7bdb39..307adc386 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -57,6 +57,8 @@ use crate::world::{Outbound, SessionTx}; /// /// Every field here is session relay state used by the shared callbacks' writer jobs. pub(crate) struct Viewer { + /// Cleared when this World Session leaves. Queued Relays must not reach its replacement. + pub(crate) active: std::sync::atomic::AtomicBool, pub(crate) session: SessionId, pub(crate) self_guid: u64, /// The account identity stamped on identity-addressed event rows. @@ -161,6 +163,7 @@ impl ViewerRegistry { fn register(&mut self, viewer: Arc, shard: ShardId) { let session = viewer.session; if let Some(previous) = self.by_session.remove(&session) { + previous.viewer.active.store(false, Ordering::Release); self.remove_indexes(session, &previous); } self.session_of_owner.insert(viewer.self_guid, session); @@ -178,6 +181,7 @@ impl ViewerRegistry { let Some(registered) = self.by_session.remove(&session) else { return; }; + registered.viewer.active.store(false, Ordering::Release); self.remove_indexes(session, ®istered); } @@ -435,8 +439,8 @@ impl WorldView { pub(crate) fn remove_viewer(&self, session: SessionId) { let mut registry = self.viewers.write().unwrap(); - self.spatial.remove_viewer(session); registry.remove(session); + self.spatial.remove_viewer(session); } /// Geometric recipients plus the row owner's session. The owner lookup lives in the viewer @@ -1378,8 +1382,7 @@ fn encounter_equip_changed(view: &WorldView, shard: ShardId, row: &EncounterEqui continue; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::encounter_equip_outbound(&viewer, &row, cleared) }); } @@ -1398,8 +1401,7 @@ fn offer_create_job( return; }; let (view, row) = (view.clone(), row.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::offer_peer_create_for(&coord, &view, shard, &viewer, &row) }); } @@ -1418,8 +1420,7 @@ fn update_job( return; }; let (view, old, new) = (view.clone(), old.clone(), new.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_entity_update(&coord, &view, shard, &viewer, &old, &new) }); } @@ -1428,8 +1429,7 @@ fn destroy_job(view: &WorldView, session: SessionId, guid: u64, owner_guid: u64) let Some(viewer) = view.viewer(session) else { return; }; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_peer_destroy(&viewer, guid, owner_guid) }); } @@ -1449,9 +1449,8 @@ fn gameobject_appeared(view: &WorldView, shard: ShardId, row: &GameObject) { let still: HashSet = recipients.iter().map(|v| v.session).collect(); for viewer in view.cell_audience(shard, Some(old_key), BOX_HALF_SPAN, &[]) { if !still.contains(&viewer.session) { - let tx = viewer.tx.clone(); let guid = row.guid; - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_gameobject_destroy(&viewer, guid) }); } @@ -1464,8 +1463,7 @@ fn gameobject_appeared(view: &WorldView, shard: ShardId, row: &GameObject) { let row = Arc::new(row.clone()); for viewer in recipients { let (coord, row) = (coord.clone(), row.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_gameobject_create(&coord, &viewer, &row) }); } @@ -1479,10 +1477,9 @@ fn gameobject_vanished(view: &WorldView, shard: ShardId, guid: u64) { return; }; for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { - let tx = viewer.tx.clone(); // DESTROY stays ungated (the `on_melee_delete` precedent): SMSG_DESTROY_OBJECT for a guid // the client never created is a client no-op, and gating it risks a stale object. - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_gameobject_destroy(&viewer, guid) }); } @@ -1540,14 +1537,16 @@ fn maybe_queue_motion_flush(_view: &WorldView, viewer: &Arc) { if viewer.motion_pending.flush_queued.swap(true, AtOrd::AcqRel) { return; // a queued flush will carry the upsert that just landed } - let tx = viewer.tx.clone(); - if super::subscriptions::shed_motion_at_depth(tx.depth()) { + if super::subscriptions::shed_motion_at_depth(viewer.tx.depth()) { super::subscriptions::MOTION_DROPPED.fetch_add(1, Ordering::Relaxed); - viewer.motion_pending.flush_queued.store(false, AtOrd::Release); + viewer + .motion_pending + .flush_queued + .store(false, AtOrd::Release); return; } let viewer = viewer.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { // Clear BEFORE draining: an upsert racing the drain re-queues a follow-up flush (possibly // empty — harmless) instead of being stranded behind a still-set flag. viewer @@ -1599,8 +1598,9 @@ fn roll_appeared(view: &WorldView, shard: ShardId, row: &RollEvent) { let row = Arc::new(row.clone()); for viewer in viewers_on_shard(view, shard) { let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::relay_roll(&row)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::relay_roll(&row) + }); } } @@ -1615,8 +1615,9 @@ fn dynobj_appeared(view: &WorldView, shard: ShardId, row: &DynamicObject) { continue; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::relay_dynobj_create(&row)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::relay_dynobj_create(&row) + }); } } @@ -1626,8 +1627,9 @@ fn dynobj_appeared(view: &WorldView, shard: ShardId, row: &DynamicObject) { fn dynobj_vanished(view: &WorldView, shard: ShardId, row: &DynamicObject) { let guid = row.guid; for viewer in viewers_on_shard(view, shard) { - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::relay_destroy_object(guid)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::relay_destroy_object(guid) + }); } } @@ -1638,8 +1640,7 @@ fn taxi_spline(view: &WorldView, shard: ShardId, row: &TaxiPassengerSpline) { let row = row.clone(); for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[row.character_guid]) { let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::taxi_spline_outbound(&viewer.created, viewer.self_guid, &row) }); } @@ -1652,8 +1653,7 @@ fn teleport_appeared(view: &WorldView, shard: ShardId, row: &TeleportEvent) { return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::teleport_event_outbound(&row) }); } @@ -1666,8 +1666,7 @@ fn addon_message_appeared(view: &WorldView, shard: ShardId, row: &AddonMessage) return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::addon_message_outbound(viewer.self_guid, &row) }); } @@ -1680,8 +1679,9 @@ fn xp_appeared(view: &WorldView, shard: ShardId, row: &XpEvent) { return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::xp_event_outbound(&row)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::xp_event_outbound(&row) + }); } /// Level-up rows are delivered to the one viewer bound to their recipient identity. @@ -1692,8 +1692,7 @@ fn levelup_appeared(view: &WorldView, shard: ShardId, row: &LevelupEvent) { return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::levelup_event_outbound(&row) }); } @@ -1703,8 +1702,8 @@ fn quest_inserted(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &C let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; - let (coord, self_guid, tx) = (coord.clone(), viewer.self_guid, viewer.tx.clone()); - enqueue(&tx, move || { + let (coord, self_guid) = (coord.clone(), viewer.self_guid); + enqueue(viewer.clone(), move |_| { let guard = coord.0.coord(); super::subscriptions::quest_log_outbound(&guard.conn.db, self_guid) .into_iter() @@ -1723,14 +1722,8 @@ fn quest_updated( let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; - let (coord, self_guid, old, row, tx) = ( - coord.clone(), - viewer.self_guid, - old.clone(), - row.clone(), - viewer.tx.clone(), - ); - enqueue(&tx, move || { + let (coord, self_guid, old, row) = (coord.clone(), viewer.self_guid, old.clone(), row.clone()); + enqueue(viewer.clone(), move |_| { let guard = coord.0.coord(); super::subscriptions::quest_update_outbound(&guard.conn.db, self_guid, &old, &row) }); @@ -1741,8 +1734,8 @@ fn quest_deleted(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &Ch let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; - let (coord, self_guid, tx) = (coord.clone(), viewer.self_guid, viewer.tx.clone()); - enqueue(&tx, move || { + let (coord, self_guid) = (coord.clone(), viewer.self_guid); + enqueue(viewer.clone(), move |_| { let guard = coord.0.coord(); super::subscriptions::quest_log_outbound(&guard.conn.db, self_guid) .into_iter() @@ -1762,8 +1755,9 @@ fn exploration_appeared( return; }; let (row, coord) = (row.clone(), coord.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || exploration_outbound(&coord, &viewer, &row)); + enqueue(viewer.clone(), move |viewer| { + exploration_outbound(&coord, &viewer, &row) + }); } fn exploration_outbound( @@ -1819,8 +1813,7 @@ fn reputation_appeared(view: &WorldView, shard: ShardId, row: &PlayerReputation) return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || reputation_outbound(&row)); + enqueue(viewer.clone(), move |_| reputation_outbound(&row)); } /// Hunter pet descriptors are idempotent VALUES, so replay re-sends are state-sync, not toasts. @@ -1829,8 +1822,9 @@ fn hunter_pet_appeared(view: &WorldView, shard: ShardId, row: &HunterPetProtocol return; }; let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::hunter_pet_outbound(&row)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::hunter_pet_outbound(&row) + }); } fn reputation_outbound(row: &PlayerReputation) -> Vec { @@ -1853,8 +1847,7 @@ fn item_owner_job( let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(owner_guid)) else { return; }; - let tx = viewer.tx.clone(); - enqueue(&tx, move || job(viewer)); + enqueue(viewer, job); } fn item_inserted(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &ItemInstance) { @@ -1908,7 +1901,7 @@ fn rest_state_appeared(view: &WorldView, row: &RestStateEvent) { return; }; let (self_guid, bytes) = (row.character_guid, row.player_bytes_2); - enqueue(&viewer.tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::relay_rest_state(self_guid, bytes) }); } @@ -1928,7 +1921,7 @@ fn breath_relay_appeared(view: &WorldView, row: &BreathRelayEvent) { row.duration_ms, row.damage, ); - enqueue(&viewer.tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::relay_breath_event(guid, kind, remaining, duration, damage) }); } @@ -1941,8 +1934,7 @@ fn skill_changed(view: &WorldView, shard: ShardId, row: &PlayerSkill) { return; }; let (line, cur, max) = (row.skill_line, row.current, row.max_rank); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_skill(&viewer, line, cur, max) }); } @@ -1958,8 +1950,7 @@ fn corpse_appeared(view: &WorldView, shard: ShardId, row: &Corpse) { } let row = row.clone(); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::corpse_create_outbound(self_guid, &row) }); } @@ -1974,8 +1965,9 @@ fn corpse_changed(view: &WorldView, shard: ShardId, row: &Corpse) { continue; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::relay_corpse_update(&row)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::relay_corpse_update(&row) + }); } } @@ -1984,8 +1976,9 @@ fn corpse_changed(view: &WorldView, shard: ShardId, row: &Corpse) { fn corpse_vanished(view: &WorldView, shard: ShardId, row: &Corpse) { let guid = row.guid; for viewer in viewers_on_shard(view, shard) { - let tx = viewer.tx.clone(); - enqueue(&tx, move || super::subscriptions::relay_destroy_object(guid)); + enqueue(viewer.clone(), move |_| { + super::subscriptions::relay_destroy_object(guid) + }); } } @@ -2019,9 +2012,8 @@ fn combat_event_appeared(view: &WorldView, shard: ShardId, row: &CombatEvent) { let row = Arc::new(row.clone()); for viewer in combat_audience(view, shard, &row) { let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { - super::subscriptions::combat_event_outbound(&viewer.tx, &viewer.created, &row) + enqueue(viewer.clone(), move |viewer| { + super::subscriptions::combat_event_outbound(&viewer, &row) }); } } @@ -2041,8 +2033,7 @@ fn melee_engaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { for viewer in melee_audience(view, shard, &row) { let row = row.clone(); let created = viewer.created.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::melee_engage_outbound(&created, &row) }); } @@ -2055,8 +2046,7 @@ fn melee_disengaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { for viewer in melee_audience(view, shard, &row) { let row = row.clone(); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::melee_disengage_outbound(self_guid, &row) }); } @@ -2074,8 +2064,7 @@ fn resurrect_offered(view: &WorldView, row: &ResurrectRequest) { return; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::resurrect_request_outbound(&row) }); } @@ -2094,8 +2083,7 @@ fn whisper_appeared(view: &WorldView, row: &WhisperEvent) { return; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::whisper_event_outbound(&row) }); } @@ -2112,8 +2100,7 @@ fn system_message_appeared(view: &WorldView, row: &SystemMessageEvent) { return; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::system_message_event_outbound(&row) }); } @@ -2131,8 +2118,7 @@ fn trade_event_appeared(view: &WorldView, row: &TradeEvent) { return; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::trade_event_outbound(&row) }); } @@ -2149,8 +2135,7 @@ fn duel_event_appeared(view: &WorldView, coord: &Coordinator, row: &DuelEvent) { { let row = row.clone(); let coord = coord.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { let template = (row.kind == lyracore_shared::duel::event_kind::REQUESTED) .then(|| coord.gameobject_template(row.flag_entry).ok().flatten()) .flatten(); @@ -2176,8 +2161,7 @@ fn duel_event_appeared(view: &WorldView, coord: &Coordinator, row: &DuelEvent) { continue; } let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::duel_winner_outbound(&row) }); } @@ -2198,8 +2182,7 @@ fn group_event_appeared(view: &WorldView, coord: &Coordinator, row: &GroupEvent) } let (row, coord) = (row.clone(), coord.clone()); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::group_event_outbound(&coord, self_guid, &row) }); } @@ -2214,8 +2197,7 @@ fn group_member_mirror_changed(view: &WorldView, coord: &Coordinator, row: &Grou return; }; let coord = coord.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::loot_tag_flags_after_membership_change(&coord, &viewer) }); } @@ -2242,8 +2224,7 @@ fn aura_applied( let row = Arc::new(row.clone()); for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::aura_insert_outbound( &coord, &view, @@ -2259,12 +2240,17 @@ fn aura_applied( } /// An aura changed → per viewer: array sync + refresh-only duration + self-only packets. -fn aura_updated(view: &Arc, coord: &Coordinator, shard: ShardId, row: &Aura, expires_changed: bool) { +fn aura_updated( + view: &Arc, + coord: &Coordinator, + shard: ShardId, + row: &Aura, + expires_changed: bool, +) { let row = Arc::new(row.clone()); for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::aura_update_outbound( &coord, &view, @@ -2290,8 +2276,7 @@ fn aura_removed( let row = Arc::new(row.clone()); for viewer in aura_audience(view, shard, &row) { let (row, coord, view) = (row.clone(), coord.clone(), view.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::aura_delete_outbound( &coord, &view, @@ -2323,8 +2308,7 @@ fn cast_event_appeared(view: &WorldView, shard: ShardId, row: &SpellCastEvent) { for viewer in cast_audience(view, shard, &row) { let row = row.clone(); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::cast_event_outbound(self_guid, &row) }); } @@ -2345,8 +2329,7 @@ fn impact_appeared(view: &WorldView, shard: ShardId, row: &SpellImpactEvent) { let row = Arc::new(row.clone()); for viewer in impact_audience(view, shard, &row) { let row = row.clone(); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::impact_event_outbound(&row) }); } @@ -2368,8 +2351,7 @@ fn emote_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &E let row = Arc::new(row.clone()); for viewer in emote_audience(view, shard, &row) { let (row, coord) = (row.clone(), coord.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::emote_event_outbound(&coord, &row) }); } @@ -2393,8 +2375,7 @@ fn chat_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &Ch for viewer in chat_audience(view, shard, &row) { let (row, coord) = (row.clone(), coord.clone()); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::chat_event_outbound(&coord, self_guid, &row) }); } @@ -2407,8 +2388,7 @@ fn channel_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: for viewer in viewers_on_shard(view, shard) { let (row, coord) = (row.clone(), coord.clone()); let self_guid = viewer.self_guid; - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::channel_event_outbound(&coord, self_guid, &row) }); } @@ -2433,8 +2413,7 @@ fn weather_changed(view: &WorldView, shard: ShardId, row: &ZoneWeather) { if viewer.zone_id.load(Ordering::Relaxed) != zone_id { continue; } - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |_| { super::subscriptions::weather_outbound(view_of_row, WeatherChangeType::Smooth) }); } @@ -2483,8 +2462,8 @@ fn zone_crossed( if !viewer.enter_zone(zone_id) { return; } - let (store, tx) = (store.clone(), viewer.tx.clone()); - enqueue(&tx, move || { + let store = store.clone(); + enqueue(viewer.clone(), move |_| { super::subscriptions::zone_weather_outbound(&store, zone_id, WeatherChangeType::Instant) }); } @@ -2563,6 +2542,7 @@ mod family_audience_tests { tx: SessionTx, ) -> Arc { Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session, self_guid, bound_identity, @@ -3002,6 +2982,7 @@ mod family_audience_tests { let view = WorldView::new(true); let old = viewer(7, 9001); let replacement = Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session: old.session, self_guid: 9002, bound_identity: identity(99), @@ -3338,7 +3319,7 @@ mod family_audience_tests { .expect("the exploration owner relay"); assert!( exploration.contains("viewer_of_owner_on_shard(shard,OwnerGuid(row.character_guid))") - && exploration.contains("enqueue(&tx,move||exploration_outbound"), + && exploration.contains("enqueue(viewer.clone(),move|viewer|{exploration_outbound"), "exploration must select its owner then defer all work to that viewer's writer" ); assert!( @@ -3360,7 +3341,7 @@ mod family_audience_tests { let body = no_ws(&source[start..]); assert!( body.contains("view.viewer_of_owner_on_shard(shard,OwnerGuid(row.character_guid))") - && body.contains("enqueue(&tx,move||") + && body.contains("enqueue(viewer.clone(),move|viewer|") && body.contains("coord.0.coord()"), "{handler} must select one shard-scoped owner and defer cache reads and packet construction to its writer job" ); @@ -3522,14 +3503,20 @@ mod family_audience_tests { // The pump→writer hand-off // =============================================================================================== -/// Push one unit of deferred relay work onto a session's writer queue. -/// -/// This is the ONLY thing the shared pump does per recipient — a boxed closure and a channel send. -/// Everything the closure does (gates, cache reads, encoding) happens on that session's own writer -/// thread, so a slow or panicking relay costs exactly one session, which is the property the -/// per-player connections used to give us for free. -fn enqueue(tx: &SessionTx, job: impl FnOnce() -> Vec + Send + 'static) { - let _ = tx.send(Outbound::Job(Box::new(job))); +/// Queue work for one World Session. The socket writer checks its lifetime before running it. +/// Jobs already running finish before the destination entry batch on the same FIFO writer; +/// jobs selected by an old callback but enqueued after that batch are discarded here. +pub(super) fn enqueue( + viewer: Arc, + job: impl FnOnce(Arc) -> Vec + Send + 'static, +) { + let tx = viewer.tx.clone(); + let _ = tx.send(Outbound::Job(Box::new(move || { + if !viewer.active.load(Ordering::Acquire) { + return Vec::new(); + } + job(viewer) + }))); } // =============================================================================================== @@ -3564,7 +3551,7 @@ pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, }) .unwrap_or(0); let viewer = viewer.clone(); - enqueue(&viewer.tx.clone(), move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_peer_destroy(&viewer, guid, owner_guid) }); } @@ -3584,7 +3571,7 @@ pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, continue; }; let (view, viewer, row) = (view.clone(), viewer.clone(), Arc::new(row)); - enqueue(&viewer.tx.clone(), move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::offer_peer_create_for(&coord, &view, shard, &viewer, &row) }); } @@ -3592,7 +3579,7 @@ pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, { for (guid, _) in delta.game_objects.left { let viewer = viewer.clone(); - enqueue(&viewer.tx.clone(), move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_gameobject_destroy(&viewer, guid) }); } @@ -3604,7 +3591,7 @@ pub(crate) fn recenter(view: &Arc, viewer: &Arc, map_id: u32, continue; }; let (viewer, row) = (viewer.clone(), Arc::new(row)); - enqueue(&viewer.tx.clone(), move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::relay_gameobject_create(&coord, &viewer, &row) }); } @@ -3715,8 +3702,7 @@ fn reconcile_shard( if view.spatial.can_see(layer, viewer.session, guid) { continue; } - let tx = viewer.tx.clone(); - enqueue(&tx, move || match layer { + enqueue(viewer.clone(), move |viewer| match layer { EntityLayer::WorldEntity => { super::subscriptions::relay_peer_destroy(&viewer, guid, owner_guid) } @@ -3735,8 +3721,7 @@ fn reconcile_shard( .entity_cell_on_shard(EntityLayer::WorldEntity, target_guid, shard); for viewer in view.cell_audience(shard, key, BOX_HALF_SPAN, &[target_guid]) { let (coord, view, previous) = (coord.clone(), view.clone(), previous.clone()); - let tx = viewer.tx.clone(); - enqueue(&tx, move || { + enqueue(viewer.clone(), move |viewer| { super::subscriptions::aura_snapshot_outbound( &coord, &view, diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index ed4f2cbc1..0a139ed84 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -36,6 +36,7 @@ fn identity(guid: u64) -> spacetimedb_sdk::Identity { fn viewer(session: SessionId, self_guid: u64, tx: SessionTx) -> Arc { Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session, self_guid, bound_identity: identity(self_guid), @@ -754,9 +755,11 @@ fn an_old_shard_cannot_move_or_change_skills_on_a_destination_viewer() { fn old_shard_combat_cannot_address_a_transferred_character() { let mut rng = Rng::new(74); let realm = spread_realm(&mut rng); - let transferred = &realm.viewers[0]; + let source = &realm.viewers[0]; + let transferred = viewer(1000, source.self_guid, source.tx.clone()); let cast = cast(&realm, transferred.self_guid, transferred.self_guid); let combat = combat(&realm, transferred.self_guid, transferred.self_guid); + realm.view.remove_viewer(source.session); realm .view .add_viewer_on_shard(transferred.clone(), CellKey::at(1, 0, 4, 4), 1); @@ -793,6 +796,66 @@ fn old_shard_combat_cannot_address_a_transferred_character() { )); } +#[test] +fn jobs_selected_before_worldport_cannot_reach_the_replacement_world_session() { + use wow_world_messages::vanilla::opcodes::ServerOpcodeMessage; + + let view = Arc::new(WorldView::new(true)); + let (tx, rx) = SessionTx::with_depth(0); + let arrival = crate::codec::EntityView { + guid: PLAYER_BASE, + ..Default::default() + }; + let old_registration = super::super::subscriptions::PlayerSubscriptions::registered_for_test( + view.clone(), + PLAYER_BASE, + &arrival, + tx.clone(), + ); + let selected = view.viewer_of_owner(OwnerGuid(PLAYER_BASE)).unwrap(); + let row = melee(PLAYER_BASE, CREATURE_BASE); + melee_engaged(&view, 0, &row); + drop(old_registration); + + // WORLDPORT writes destination entry on the same socket before registering its fresh Viewer. + tx.send(Outbound::Raw { + opcode: 0xCAFE, + body: vec![], + }) + .unwrap(); + let _destination = super::super::subscriptions::PlayerSubscriptions::registered_for_test( + view.clone(), + PLAYER_BASE, + &arrival, + tx, + ); + let delayed_row = row.clone(); + enqueue(selected, move |viewer| { + super::super::subscriptions::melee_engage_outbound(&viewer.created, &delayed_row) + }); + melee_engaged(&view, 0, &row); + + let Outbound::Job(before_entry) = rx.try_recv().unwrap() else { + panic!("old Relay job"); + }; + assert!(before_entry().is_empty()); + assert!(matches!( + rx.try_recv().unwrap(), + Outbound::Raw { opcode: 0xCAFE, .. } + )); + let Outbound::Job(after_entry) = rx.try_recv().unwrap() else { + panic!("late old Relay job"); + }; + assert!(after_entry().is_empty()); + let Outbound::Job(current) = rx.try_recv().unwrap() else { + panic!("current Relay job"); + }; + assert!(matches!( + current().as_slice(), + [Outbound::One(ServerOpcodeMessage::SMSG_ATTACKSTART(_))] + )); +} + #[test] fn spatial_candidates_also_stay_on_the_event_shard() { let view = WorldView::new(true); diff --git a/gateway/src/stdb/world_view/relay_bench.rs b/gateway/src/stdb/world_view/relay_bench.rs index 1de33e737..5d27cfd0a 100644 --- a/gateway/src/stdb/world_view/relay_bench.rs +++ b/gateway/src/stdb/world_view/relay_bench.rs @@ -39,6 +39,7 @@ fn viewer(session: SessionId, self_guid: u64, tx: SessionTx) -> Arc { let mut identity = [0u8; 32]; identity[..8].copy_from_slice(&self_guid.to_le_bytes()); Arc::new(Viewer { + active: std::sync::atomic::AtomicBool::new(true), session, self_guid, bound_identity: spacetimedb_sdk::Identity::from_byte_array(identity), From a9c94ce7c283da44fc3c9be7e7cd5519df1ff13e Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Fri, 4 Sep 2026 23:48:59 +0200 Subject: [PATCH 6/9] fix(gateway): retire old pump before publishing replacement cache --- gateway/src/stdb/connection.rs | 182 ++++++++++++++---- gateway/src/stdb/subscriptions.rs | 12 +- gateway/src/stdb/world_index.rs | 35 +++- gateway/src/stdb/world_view.rs | 171 +++------------- .../stdb/world_view/cell_audience_tests.rs | 76 -------- 5 files changed, 207 insertions(+), 269 deletions(-) diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index 4b430e110..3cf3ef2ee 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -59,7 +59,7 @@ pub(crate) struct LiveConn { /// connection generation, so a watchdog replacement cannot fail calls on the fresh socket. pub(crate) reducer_completion: Arc, /// Keeps the SDK message-pump thread alive for the connection's lifetime. - _pump: std::thread::JoinHandle<()>, + _pump: Option>, pump_work: tokio::sync::mpsc::UnboundedSender, /// Keeps this role's subscription active for the connection's lifetime. _sub: SubscriptionHandle, @@ -117,19 +117,23 @@ impl LiveConn { /// A connection needs its transport, subscription, and pump. A Module republish can /// invalidate the subscription while the socket remains connected. fn is_healthy(&self) -> bool { - self.conn.is_active() && self._sub.is_active() && !self._pump.is_finished() + self.conn.is_active() + && self._sub.is_active() + && self._pump.as_ref().is_some_and(|pump| !pump.is_finished()) } } -/// Install a fresh resource before post-install work, then tear down the replaced resource. This -/// generic seam keeps the ordering and lock lifetime deterministic without constructing SDK types. +/// Retire callbacks before publishing a fresh resource, then run setup and release the old resource. +/// This external connection seam exposes publication order without constructing SDK types. fn replace_slot_ordered( slot: &RwLock, fresh: T, role_label: &str, + retire: impl FnOnce(&RwLock), post_install: impl FnOnce(), teardown: impl FnOnce(T), ) { + retire(slot); let old = { let mut live = slot.write().unwrap_or_else(|poisoned| { log::error!("{role_label} live-slot lock poisoned during replacement; recovering"); @@ -141,8 +145,9 @@ fn replace_slot_ordered( teardown(old); } -/// Replace one live SDK generation. The fresh generation is visible before role-specific work; -/// that work, disconnect, and pump join all run without the live-slot write lock. +/// Drain the old pump before exposing the fresh cache. A fresh source cache can advance a +/// Transfer to its destination, so no old source callback may update the shared indexes afterward. +/// The slot lock is released before disconnect or join; callbacks can finish their cache reads. fn replace_live_conn( slot: &RwLock, fresh: LiveConn, @@ -153,22 +158,29 @@ fn replace_live_conn( slot, fresh, role_label, + |slot| { + let (conn, pump) = { + let mut live = slot + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + (live.conn.clone(), live._pump.take()) + }; + if let Err(error) = conn.disconnect() { + if !matches!(error, spacetimedb_sdk::Error::Disconnected) { + log::warn!("{role_label} old connection disconnect failed: {error}"); + } + } + if let Some(pump) = pump { + if let Err(error) = pump.join() { + log::warn!("{role_label} old pump thread panicked: {error:?}"); + } + } + }, || { log::info!("{role_label} replacement installed"); post_install(); }, - |old| { - if let Err(error) = old.conn.disconnect() { - log::warn!( - "{role_label} old connection disconnect failed during teardown: {error}" - ); - } - if let Err(error) = old._pump.join() { - log::warn!("{role_label} old pump thread panicked during teardown: {error:?}"); - } else { - log::info!("{role_label} old connection teardown complete"); - } - }, + drop, ); } @@ -548,7 +560,7 @@ fn connect_subscribed( Ok(LiveConn { conn, reducer_completion, - _pump: pump, + _pump: Some(pump), pump_work, _sub: sub, }) @@ -986,19 +998,10 @@ fn connect_blocking( /// How often the coordinator watchdog polls connection liveness. const COORDINATOR_WATCHDOG_POLL: Duration = Duration::from_secs(3); -/// Background watchdog: detect a dropped coordinator connection and rebuild it IN PLACE so the gateway -/// self-heals across a SpacetimeDB migration / network blip — no manual restart. Each poll it checks -/// socket, subscription, and pump liveness through [`LiveConn::is_healthy`]. -/// A migration can invalidate the SUBSCRIPTION while the socket stays up -/// (`conn.is_active()` stays true, and the SDK's subscription `on_disconnect` is a no-op), and a raw -/// socket drop leaves the subscription's status at `Applied`. Recovery also replaces a stopped pump. On a drop -/// it rebuilds a fresh connection + resubscribes OFF-LOCK (the up-to-15s apply must not block readers), -/// swaps it in under the write lock (instant), then tears the OLD connection down off-lock (disconnect + -/// join its pump thread, so it's reaped rather than detached-and-leaked — on the subscription-death path -/// the old socket was still live, so its pump is still running). While rebuilding, cache reads return the -/// stale/empty view (infallible `Option` reads → `None`/missing row, e.g. `player_login`'s "not visible" -/// path → a clean relog, never a panic); once the swap lands, every reader picks up the fresh connection -/// on its next access. Retries on failure; never panics. +/// Rebuild a Coordinator when its transport, subscription, or pump stops. A Module migration +/// can invalidate the subscription while the socket remains connected. +/// Build and subscribe off-lock, drain the old pump, then publish and arm the fresh connection. +/// Readers retain the old cache during recovery; failed connections retry on the next poll. fn spawn_coordinator_watchdog(inner: Arc) -> std::thread::JoinHandle<()> { std::thread::Builder::new() .name("stdb-coordinator-watchdog".into()) @@ -1027,8 +1030,8 @@ fn spawn_coordinator_watchdog(inner: Arc) -> std::thread::Join Ok(fresh) => { // Re-arm any relay with no per-login point to self-heal through — MUST run // after the install: each hook re-reads `inner.coord()` and must see fresh. - // `replace_live_conn` also guarantees hooks and teardown run without its write - // guard, with teardown ordered after every hook. + // The old pump has finished before publication. Hooks read the fresh cache + // without holding the slot write lock. replace_live_conn(&inner.live, fresh, &role_label, || { let hooks = inner.on_reconnect.lock().unwrap().clone(); for hook in hooks { @@ -1452,9 +1455,12 @@ mod coordinator_query_tests { #[cfg(test)] mod live_replacement_tests { - use super::replace_slot_ordered; + use super::{pump_messages, replace_slot_ordered}; + use crate::stdb::world_index::{CellKey, EntityLayer, WorldIndex}; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Mutex, RwLock}; + use std::sync::{mpsc, Arc, Mutex, RwLock}; + use std::thread::JoinHandle; + use std::time::Duration; #[test] fn replacement_installs_before_post_install_and_tears_down_unlocked() { @@ -1466,6 +1472,7 @@ mod live_replacement_tests { &slot, "fresh", "test database coordinator", + |_| {}, || { let live = slot .try_read() @@ -1488,6 +1495,107 @@ mod live_replacement_tests { assert_eq!(post_install_calls.load(Ordering::Relaxed), 1); assert_eq!(*events.lock().unwrap(), ["post-install", "teardown"]); } + + #[test] + fn a_replacement_cache_cannot_advance_transfer_past_an_old_row_callback() { + struct ExternalConnection { + escrow_visible: bool, + pump: Option>, + } + + let source = 0; + let destination = 1; + let guid = 7; + let source_cell = CellKey::at(0, 0, 0, 0); + let destination_cell = CellKey::at(1, 0, 0, 0); + for inserting in [true, false] { + let index = Arc::new(WorldIndex::new(true)); + if !inserting { + index.upsert_entity(EntityLayer::WorldEntity, guid, source_cell, source, 0); + } + let slot = Arc::new(RwLock::new(ExternalConnection { + escrow_visible: false, + pump: None, + })); + let (entered, callback_entered) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let callback_slot = slot.clone(); + let callback_index = index.clone(); + let pump = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let (_setup, work) = tokio::sync::mpsc::unbounded_channel(); + let result = runtime.block_on(pump_messages(&(), work, || async { + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!( + !callback_slot.read().unwrap().escrow_visible, + "an old callback must retain the old cache until it finishes" + ); + callback_index.upsert_entity( + EntityLayer::WorldEntity, + guid, + source_cell, + source, + 0, + ); + Err::<(), ()>(()) + })); + assert_eq!(result, Err(())); + }); + slot.write().unwrap().pump = Some(pump); + callback_entered + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + + let (retiring, retirement_started) = mpsc::channel(); + let replacement_slot = slot.clone(); + let replacement_index = index.clone(); + let replacement = std::thread::spawn(move || { + replace_slot_ordered( + &replacement_slot, + ExternalConnection { + escrow_visible: true, + pump: None, + }, + "source Shard", + |slot| { + let pump = slot.write().unwrap().pump.take().unwrap(); + retiring.send(()).unwrap(); + pump.join().unwrap(); + }, + || { + assert!(replacement_slot.read().unwrap().escrow_visible); + replacement_index.upsert_entity( + EntityLayer::WorldEntity, + guid, + destination_cell, + destination, + 0, + ); + }, + drop, + ); + }); + retirement_started + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + let published_before_callback_finished = slot.read().unwrap().escrow_visible; + release.send(()).unwrap(); + replacement.join().unwrap(); + + assert!(!published_before_callback_finished); + assert_eq!( + index.shard_of(EntityLayer::WorldEntity, guid), + Some(destination) + ); + assert_eq!( + index.entity_cell(EntityLayer::WorldEntity, guid), + Some(destination_cell) + ); + } + } } #[cfg(test)] @@ -1515,7 +1623,7 @@ mod pump_tests { let _ = done.send(()); })) .unwrap(); - let pumping = pump_messages(&cache, rx, || std::future::pending::>()); + let pumping = pump_messages(&cache, rx, std::future::pending::>); tokio::pin!(pumping); tokio::select! { _ = &mut pumping => panic!("the external connection is still open"), diff --git a/gateway/src/stdb/subscriptions.rs b/gateway/src/stdb/subscriptions.rs index 6fd38f0c6..34b8d107e 100644 --- a/gateway/src/stdb/subscriptions.rs +++ b/gateway/src/stdb/subscriptions.rs @@ -5541,11 +5541,11 @@ mod tests { let body = decommented(top_level_fn_body_of("world_view.rs", "arm_realm_private")); let compact: String = body.chars().filter(|c| !c.is_whitespace()).collect(); assert!( - compact.contains("wire_insert_live(db.game_whisper_event(),\"realm.game_whisper_event.insert\",&view,None,|v,row|whisper_appeared(v,row),);"), + compact.contains("wire_insert_live(db.game_whisper_event(),\"realm.game_whisper_event.insert\",&view,|v,row|whisper_appeared(v,row),);"), "arm_realm_private no longer relays realm-core whispers through `whisper_appeared`" ); assert!( - compact.contains("wire_insert_live(db.game_group_event(),\"realm.game_group_event.insert\",&view,None,move|v,row|group_event_appeared(v,&coord,row),);"), + compact.contains("wire_insert_live(db.game_group_event(),\"realm.game_group_event.insert\",&view,move|v,row|group_event_appeared(v,&coord,row),);"), "arm_realm_private no longer relays realm-core group events through \ `group_event_appeared` (which also carries the QUEST_SHARE detail JOIN through a \ WORLD handle — realm-core's cache has no quest catalogue)" @@ -6271,7 +6271,13 @@ mod tests { /// freeze after their first step, or never move at all. Same for the creature-leg twin. #[test] fn both_halves_of_the_motion_and_spline_relays_are_registered_286() { - let arm: String = decommented(top_level_fn_body_of("world_view.rs", "register_shard_callbacks")).chars().filter(|c| !c.is_whitespace()).collect(); + let arm: String = decommented(top_level_fn_body_of( + "world_view.rs", + "register_shard_callbacks", + )) + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); // #490 factored every registration through `wire_insert`/`wire_update` (world_view.rs), // so the literal `.on_insert(`/`.on_update(` chain off the table handle is gone from // `arm_shard`'s own body — what's left to scan for is the (helper, table, label) triple diff --git a/gateway/src/stdb/world_index.rs b/gateway/src/stdb/world_index.rs index aa70978db..624ade461 100644 --- a/gateway/src/stdb/world_index.rs +++ b/gateway/src/stdb/world_index.rs @@ -479,7 +479,11 @@ impl WorldIndex { for cell in cells.difference(other) { if let Some(guids) = entities.cell_entities.get(cell) { for guid in guids { - let shard = entities.entity_cell.get(guid).map(|(_, s, _)| *s).unwrap_or(0); + let shard = entities + .entity_cell + .get(guid) + .map(|(_, s, _)| *s) + .unwrap_or(0); out.push((*guid, shard)); } } @@ -909,16 +913,29 @@ mod tests { let b = CellKey::at(0, 0, 40, 40); // far outside a's box index.upsert_entity(EntityLayer::WorldEntity, 5, a, 0, 0); index.add_viewer(1, a); - assert_eq!(index_visible(&index, EntityLayer::WorldEntity, 1), HashSet::from([5])); - assert_eq!(index.upsert_entity(EntityLayer::WorldEntity, 5, b, 0, 0), Some(a)); + assert_eq!( + index_visible(&index, EntityLayer::WorldEntity, 1), + HashSet::from([5]) + ); + assert_eq!( + index.upsert_entity(EntityLayer::WorldEntity, 5, b, 0, 0), + Some(a) + ); assert!( index_visible(&index, EntityLayer::WorldEntity, 1).is_empty(), "stale cell membership" ); assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 5), Some(b)); - assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5, 0), Some((b, 0))); + assert_eq!( + index.remove_entity(EntityLayer::WorldEntity, 5, 0), + Some((b, 0)) + ); assert_eq!(index.entity_cell(EntityLayer::WorldEntity, 5), None); - assert_eq!(index.remove_entity(EntityLayer::WorldEntity, 5, 0), None, "removal is idempotent"); + assert_eq!( + index.remove_entity(EntityLayer::WorldEntity, 5, 0), + None, + "removal is idempotent" + ); } #[test] @@ -980,7 +997,13 @@ mod tests { fn the_aoi_off_hatch_sees_the_whole_partition_and_nothing_outside_it() { let index = WorldIndex::new(false); index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0, 0); - index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 500, 500), 0, 0); + index.upsert_entity( + EntityLayer::WorldEntity, + 2, + CellKey::at(0, 0, 500, 500), + 0, + 0, + ); index.upsert_entity(EntityLayer::WorldEntity, 3, CellKey::at(0, 77, 0, 0), 0, 0); // another instance index.upsert_entity(EntityLayer::WorldEntity, 4, CellKey::at(1, 0, 0, 0), 0, 0); // another map index.add_viewer(1, CellKey::at(0, 0, 0, 0)); diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index 307adc386..4f17f27de 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -300,7 +300,6 @@ pub(crate) struct WorldView { /// dispatch; the order fixes the ids, so it must never be reordered after arming. shards: RwLock>, next_session: AtomicU64, - generations: Mutex>>>, } impl WorldView { @@ -311,7 +310,6 @@ impl WorldView { viewers: RwLock::new(ViewerRegistry::default()), shards: RwLock::new(Vec::new()), next_session: AtomicU64::new(1), - generations: Mutex::new(HashMap::new()), } } @@ -505,55 +503,15 @@ impl WorldView { pub(crate) fn set_shards(&self, shards: Vec) { *self.shards.write().unwrap() = shards; } - - fn with_new_generation(&self, shard: ShardId, body: impl FnOnce(ShardGeneration)) { - let current = self - .generations - .lock() - .unwrap() - .entry(shard) - .or_default() - .clone(); - let mut number = current.lock().unwrap_or_else(|p| p.into_inner()); - *number += 1; - body(ShardGeneration { - current: current.clone(), - number: *number, - }); - } - -} - -#[derive(Clone)] -struct ShardGeneration { - current: Arc>, - number: u64, -} - -impl ShardGeneration { - /// The lock also waits for an old callback already in progress before snapshot replacement. - fn run(generation: Option<&Self>, body: impl FnOnce()) { - if let Some(generation) = generation { - let current = generation.current.lock().unwrap_or_else(|p| p.into_inner()); - if *current != generation.number { - return; - } - body(); - } else { - body(); - } - } } -/// Register Relays and reconcile the cache snapshot between messages on this Shard's pump. -/// The generation lock finishes any old callback first and rejects its later callbacks. +/// Register Relays and reconcile the snapshot between messages on this Shard's pump. +/// Connection replacement joins the old pump before this connection becomes readable. pub(crate) fn arm_shard(view: Arc, coord: Coordinator, shard: ShardId) -> Result<()> { let owner = coord.clone(); coord.0.on_pump(move |conn| { - view.with_new_generation(shard, |generation| { - register_shard_callbacks(view.clone(), owner, shard, &conn.db, generation); - seed_shard_from_cache(&view, shard, &conn.db); - }); + register_shard_callbacks(view.clone(), owner, shard, &conn.db); + seed_shard_from_cache(&view, shard, &conn.db); }) } @@ -562,7 +520,6 @@ fn register_shard_callbacks( coord: Coordinator, shard: ShardId, db: &RemoteTables, - generation: ShardGeneration, ) { // ---- owner-scoped events --------------------------------------------------------------- // These callbacks address one viewer directly and survive coordinator reconnects because @@ -576,28 +533,24 @@ fn register_shard_callbacks( db.game_teleport_event(), "game_teleport_event.insert", &view, - Some(&generation), move |v, row| teleport_appeared(v, shard, row), ); wire_insert_live( db.game_addon_message(), "game_addon_message.insert", &view, - Some(&generation), move |v, row| addon_message_appeared(v, shard, row), ); wire_insert_live( db.game_xp_event(), "game_xp_event.insert", &view, - Some(&generation), move |v, row| xp_appeared(v, shard, row), ); wire_insert_live( db.game_levelup_event(), "game_levelup_event.insert", &view, - Some(&generation), move |v, row| levelup_appeared(v, shard, row), ); let quest_insert_coord = coord.clone(); @@ -605,7 +558,6 @@ fn register_shard_callbacks( db.game_character_quest(), "game_character_quest.insert", &view, - Some(&generation), move |v, row| quest_inserted(v, &quest_insert_coord, shard, row), ); let quest_update_coord = coord.clone(); @@ -613,7 +565,6 @@ fn register_shard_callbacks( db.game_character_quest(), "game_character_quest.update", &view, - Some(&generation), move |v, old, row| quest_updated(v, &quest_update_coord, shard, old, row), ); let quest_delete_coord = coord.clone(); @@ -621,7 +572,6 @@ fn register_shard_callbacks( db.game_character_quest(), "game_character_quest.delete", &view, - Some(&generation), move |v, row| quest_deleted(v, &quest_delete_coord, shard, row), ); { @@ -630,7 +580,6 @@ fn register_shard_callbacks( db.game_character_explored(), "game_character_explored.insert", &view, - Some(&generation), move |v, row| exploration_appeared(v, &coord, shard, row), ); } @@ -638,28 +587,24 @@ fn register_shard_callbacks( db.game_player_reputation(), "game_player_reputation.insert", &view, - Some(&generation), move |v, row| reputation_appeared(v, shard, row), ); wire_update( db.game_player_reputation(), "game_player_reputation.update", &view, - Some(&generation), move |v, _old, row| reputation_appeared(v, shard, row), ); wire_insert( db.game_hunter_pet_protocol(), "game_hunter_pet_protocol.insert", &view, - Some(&generation), move |v, row| hunter_pet_appeared(v, shard, row), ); wire_update( db.game_hunter_pet_protocol(), "game_hunter_pet_protocol.update", &view, - Some(&generation), move |v, old, row| { if old != row { hunter_pet_appeared(v, shard, row) @@ -672,7 +617,6 @@ fn register_shard_callbacks( db.game_item_instance(), "game_item_instance.insert", &view, - Some(&generation), move |v, row| item_inserted(v, &coord, shard, row), ); } @@ -682,7 +626,6 @@ fn register_shard_callbacks( db.game_item_instance(), "game_item_instance.update", &view, - Some(&generation), move |v, old, row| item_updated(v, &coord, shard, old, row), ); } @@ -692,7 +635,6 @@ fn register_shard_callbacks( db.game_item_instance(), "game_item_instance.delete", &view, - Some(&generation), move |v, row| item_deleted(v, &coord, shard, row), ); } @@ -702,7 +644,6 @@ fn register_shard_callbacks( db.game_world_entity(), "game_world_entity.insert", &view, - Some(&generation), move |v, row| entity_appeared(v, shard, row), ); { @@ -714,7 +655,6 @@ fn register_shard_callbacks( db.game_world_entity(), "game_world_entity.update", &view, - Some(&generation), move |v, old, new| { entity_changed(v, shard, old, new); zone_crossed(v, &coord, shard, new.guid, old.zone_id, new.zone_id); @@ -725,7 +665,6 @@ fn register_shard_callbacks( db.game_world_entity(), "game_world_entity.delete", &view, - Some(&generation), move |v, row| entity_vanished(v, shard, row.guid, row.owner_guid), ); @@ -736,21 +675,18 @@ fn register_shard_callbacks( db.game_encounter_equip(), "game_encounter_equip.insert", &view, - Some(&generation), move |v, row| encounter_equip_changed(v, shard, row, false), ); wire_update( db.game_encounter_equip(), "game_encounter_equip.update", &view, - Some(&generation), move |v, _old, row| encounter_equip_changed(v, shard, row, false), ); wire_delete( db.game_encounter_equip(), "game_encounter_equip.delete", &view, - Some(&generation), move |v, row| encounter_equip_changed(v, shard, row, true), ); @@ -759,21 +695,18 @@ fn register_shard_callbacks( db.game_gameobject(), "game_gameobject.insert", &view, - Some(&generation), move |v, row| gameobject_appeared(v, shard, row), ); wire_update( db.game_gameobject(), "game_gameobject.update", &view, - Some(&generation), move |v, _old, row| gameobject_appeared(v, shard, row), ); wire_delete( db.game_gameobject(), "game_gameobject.delete", &view, - Some(&generation), move |v, row| gameobject_vanished(v, shard, row.guid), ); @@ -782,14 +715,12 @@ fn register_shard_callbacks( db.game_entity_motion(), "game_entity_motion.insert", &view, - Some(&generation), move |v, row| motion(v, shard, row), ); wire_update( db.game_entity_motion(), "game_entity_motion.update", &view, - Some(&generation), move |v, _old, row| motion(v, shard, row), ); @@ -798,14 +729,12 @@ fn register_shard_callbacks( db.game_creature_spline(), "game_creature_spline.insert", &view, - Some(&generation), move |v, row| creature_leg(v, shard, row), ); wire_update( db.game_creature_spline(), "game_creature_spline.update", &view, - Some(&generation), move |v, _old, row| creature_leg(v, shard, row), ); @@ -814,14 +743,12 @@ fn register_shard_callbacks( db.game_taxi_passenger_spline(), "game_taxi_passenger_spline.insert", &view, - Some(&generation), move |v, row| taxi_spline(v, shard, row), ); wire_update( db.game_taxi_passenger_spline(), "game_taxi_passenger_spline.update", &view, - Some(&generation), move |v, _old, row| taxi_spline(v, shard, row), ); @@ -831,7 +758,6 @@ fn register_shard_callbacks( db.game_roll_event(), "game_roll_event.insert", &view, - Some(&generation), move |v, row| roll_appeared(v, shard, row), ); @@ -841,7 +767,6 @@ fn register_shard_callbacks( db.game_rest_state_event(), "game_rest_state_event.insert", &view, - Some(&generation), |v, row| rest_state_appeared(v, row), ); @@ -851,7 +776,6 @@ fn register_shard_callbacks( db.game_breath_relay_event(), "game_breath_relay_event.insert", &view, - Some(&generation), |v, row| breath_relay_appeared(v, row), ); @@ -862,14 +786,12 @@ fn register_shard_callbacks( db.game_dynamic_object(), "game_dynamic_object.insert", &view, - Some(&generation), move |v, row| dynobj_appeared(v, shard, row), ); wire_delete( db.game_dynamic_object(), "game_dynamic_object.delete", &view, - Some(&generation), move |v, row| dynobj_vanished(v, shard, row), ); @@ -880,21 +802,18 @@ fn register_shard_callbacks( db.game_corpse(), "game_corpse.insert", &view, - Some(&generation), move |v, row| corpse_appeared(v, shard, row), ); wire_update( db.game_corpse(), "game_corpse.update", &view, - Some(&generation), move |v, _old, row| corpse_changed(v, shard, row), ); wire_delete( db.game_corpse(), "game_corpse.delete", &view, - Some(&generation), move |v, row| corpse_vanished(v, shard, row), ); @@ -905,7 +824,6 @@ fn register_shard_callbacks( db.game_combat_event(), "game_combat_event.insert", &view, - Some(&generation), move |v, row| combat_event_appeared(v, shard, row), ); @@ -916,14 +834,12 @@ fn register_shard_callbacks( db.game_melee_attack(), "game_melee_attack.insert", &view, - Some(&generation), move |v, row| melee_engaged(v, shard, row), ); wire_delete( db.game_melee_attack(), "game_melee_attack.delete", &view, - Some(&generation), move |v, row| melee_disengaged(v, shard, row), ); @@ -937,21 +853,18 @@ fn register_shard_callbacks( db.game_resurrect_request(), "game_resurrect_request.insert", &view, - Some(&generation), |v, row| resurrect_offered(v, row), ); wire_insert_live( db.game_whisper_event(), "game_whisper_event.insert", &view, - Some(&generation), |v, row| whisper_appeared(v, row), ); wire_insert_live( db.game_system_message_event(), "game_system_message_event.insert", &view, - Some(&generation), |v, row| system_message_appeared(v, row), ); { @@ -960,7 +873,6 @@ fn register_shard_callbacks( db.game_group_event(), "game_group_event.insert", &view, - Some(&generation), move |v, row| group_event_appeared(v, &coord, row), ); } @@ -970,7 +882,6 @@ fn register_shard_callbacks( db.game_group_member(), "game_group_member.insert", &view, - Some(&generation), move |v, row| group_member_mirror_changed(v, &insert_coord, row), ); let delete_coord = coord.clone(); @@ -978,7 +889,6 @@ fn register_shard_callbacks( db.game_group_member(), "game_group_member.delete", &view, - Some(&generation), move |v, row| group_member_mirror_changed(v, &delete_coord, row), ); } @@ -986,7 +896,6 @@ fn register_shard_callbacks( db.game_trade_event(), "game_trade_event.insert", &view, - Some(&generation), |v, row| trade_event_appeared(v, row), ); { @@ -995,7 +904,6 @@ fn register_shard_callbacks( db.game_duel_event(), "game_duel_event.insert", &view, - Some(&generation), move |v, row| duel_event_appeared(v, &coord, row), ); } @@ -1009,42 +917,33 @@ fn register_shard_callbacks( { let view = view.clone(); let coord = coord.clone(); - let generation = generation.clone(); db.game_aura().on_insert(move |_ctx, row| { guarded("game_aura.insert", || { - ShardGeneration::run(Some(&generation), || { - view.auras.upsert(shard, row); - let n = view.auras.stealth_count(shard, row.target_guid); - aura_applied(&view, &coord, shard, row, n) - }) + view.auras.upsert(shard, row); + let n = view.auras.stealth_count(shard, row.target_guid); + aura_applied(&view, &coord, shard, row, n) }); }); } { let view = view.clone(); let coord = coord.clone(); - let generation = generation.clone(); db.game_aura().on_update(move |_ctx, old, row| { let expires_changed = old.expires_at != row.expires_at; guarded("game_aura.update", || { - ShardGeneration::run(Some(&generation), || { - view.auras.upsert(shard, row); - aura_updated(&view, &coord, shard, row, expires_changed) - }) + view.auras.upsert(shard, row); + aura_updated(&view, &coord, shard, row, expires_changed) }); }); } { let view = view.clone(); let coord = coord.clone(); - let generation = generation.clone(); db.game_aura().on_delete(move |_ctx, row| { guarded("game_aura.delete", || { - ShardGeneration::run(Some(&generation), || { - view.auras.remove(shard, row); - let n = view.auras.stealth_count(shard, row.target_guid); - aura_removed(&view, &coord, shard, row, n) - }) + view.auras.remove(shard, row); + let n = view.auras.stealth_count(shard, row.target_guid); + aura_removed(&view, &coord, shard, row, n) }); }); } @@ -1056,14 +955,12 @@ fn register_shard_callbacks( db.game_spell_cast_event(), "game_spell_cast_event.insert", &view, - Some(&generation), move |v, row| cast_event_appeared(v, shard, row), ); wire_insert( db.game_spell_impact_event(), "game_spell_impact_event.insert", &view, - Some(&generation), move |v, row| impact_appeared(v, shard, row), ); @@ -1077,7 +974,6 @@ fn register_shard_callbacks( db.game_emote_event(), "game_emote_event.insert", &view, - Some(&generation), move |v, row| emote_appeared(v, &coord, shard, row), ); } @@ -1087,7 +983,6 @@ fn register_shard_callbacks( db.game_chat_event(), "game_chat_event.insert", &view, - Some(&generation), move |v, row| chat_appeared(v, &coord, shard, row), ); } @@ -1097,7 +992,6 @@ fn register_shard_callbacks( db.game_channel_event(), "game_channel_event.insert", &view, - Some(&generation), move |v, row| channel_appeared(v, &coord, shard, row), ); } @@ -1110,14 +1004,12 @@ fn register_shard_callbacks( db.game_zone_weather(), "game_zone_weather.insert", &view, - Some(&generation), move |v, row| weather_changed(v, shard, row), ); wire_update( db.game_zone_weather(), "game_zone_weather.update", &view, - Some(&generation), move |v, _old, row| weather_changed(v, shard, row), ); @@ -1128,14 +1020,12 @@ fn register_shard_callbacks( db.game_player_skill(), "game_player_skill.insert", &view, - Some(&generation), move |v, row| skill_changed(v, shard, row), ); wire_update( db.game_player_skill(), "game_player_skill.update", &view, - Some(&generation), move |v, _old, row| skill_changed(v, shard, row), ); } @@ -1161,14 +1051,12 @@ pub(crate) fn arm_realm_private(view: Arc, realm: Coordinator, coord: db.game_whisper_event(), "realm.game_whisper_event.insert", &view, - None, |v, row| whisper_appeared(v, row), ); wire_insert_live( db.game_group_event(), "realm.game_group_event.insert", &view, - None, move |v, row| group_event_appeared(v, &coord, row), ); } @@ -1200,17 +1088,13 @@ fn wire_insert( table: T, label: &'static str, view: &Arc, - generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); - let generation = generation.cloned(); table.on_insert(move |_ctx, row| { - guarded(label, || { - ShardGeneration::run(generation.as_ref(), || f(&view, row)) - }); + guarded(label, || f(&view, row)); }); } @@ -1228,20 +1112,16 @@ fn wire_insert_live( table: T, label: &'static str, view: &Arc, - generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); - let generation = generation.cloned(); table.on_insert(move |ctx, row| { if is_initial_apply(&ctx.event) { return; } - guarded(label, || { - ShardGeneration::run(generation.as_ref(), || f(&view, row)) - }); + guarded(label, || f(&view, row)); }); } @@ -1250,17 +1130,13 @@ fn wire_update( table: T, label: &'static str, view: &Arc, - generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row, &T::Row) + Send + 'static, ) where T: TableWithPrimaryKey, { let view = view.clone(); - let generation = generation.cloned(); table.on_update(move |_ctx, old, new| { - guarded(label, || { - ShardGeneration::run(generation.as_ref(), || f(&view, old, new)) - }); + guarded(label, || f(&view, old, new)); }); } @@ -1269,17 +1145,13 @@ fn wire_delete( table: T, label: &'static str, view: &Arc, - generation: Option<&ShardGeneration>, f: impl Fn(&Arc, &T::Row) + Send + 'static, ) where T: Table, { let view = view.clone(); - let generation = generation.cloned(); table.on_delete(move |_ctx, row| { - guarded(label, || { - ShardGeneration::run(generation.as_ref(), || f(&view, row)) - }); + guarded(label, || f(&view, row)); }); } @@ -2023,7 +1895,12 @@ fn melee_audience(view: &WorldView, shard: ShardId, row: &MeleeAttack) -> Vec) { } } -/// Replace this shard's rows on its pump. No SDK message or old-generation callback can -/// interleave with the snapshot, and other shards retain their current rows. +/// Replace this Shard's rows between SDK messages. Its old pump has stopped, and other +/// Shards retain their current rows. fn seed_shard_from_cache(view: &Arc, shard: ShardId, db: &RemoteTables) { reconcile_shard( view, diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index 0a139ed84..9b3924af7 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -1035,82 +1035,6 @@ fn reconnect_clears_the_action_bar_of_a_pet_missing_from_the_snapshot() { ); } -#[test] -fn an_old_connection_cannot_change_a_new_shard_snapshot() { - let view = Arc::new(WorldView::new(true)); - let mut old = None; - view.with_new_generation(0, |generation| old = Some(generation)); - let old = old.unwrap(); - let cell = CellKey::at(0, 0, 0, 0); - ShardGeneration::run(Some(&old), || { - view.spatial - .upsert_entity(EntityLayer::WorldEntity, CREATURE_BASE, cell, 0, 0); - view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); - }); - view.with_new_generation(0, |_| reconcile_shard(&view, 0, vec![], vec![], vec![])); - - ShardGeneration::run(Some(&old), || { - view.spatial - .upsert_entity(EntityLayer::WorldEntity, CREATURE_BASE, cell, 0, 0); - view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); - }); - assert_eq!( - view.spatial - .entity_cell(EntityLayer::WorldEntity, CREATURE_BASE), - None - ); - assert!(view.auras.on_target(0, CREATURE_BASE).is_empty()); -} - -#[test] -fn a_reconnect_waits_for_an_old_callback_already_in_progress() { - let view = Arc::new(WorldView::new(true)); - let mut generation = None; - view.with_new_generation(0, |current| generation = Some(current)); - let generation = generation.unwrap(); - let (entered, started) = std::sync::mpsc::channel(); - let (release, released) = std::sync::mpsc::channel(); - let (attempt, attempted) = std::sync::mpsc::channel(); - let (complete, completed) = std::sync::mpsc::channel(); - std::thread::scope(|threads| { - let old_view = view.clone(); - threads.spawn(move || { - ShardGeneration::run(Some(&generation), || { - entered.send(()).unwrap(); - released.recv().unwrap(); - old_view.spatial.upsert_entity( - EntityLayer::WorldEntity, - CREATURE_BASE, - CellKey::at(0, 0, 0, 0), - 0, - 0, - ); - old_view.auras.upsert(0, &aura(1, CREATURE_BASE, A_STEALTH)); - }) - }); - started.recv().unwrap(); - threads.spawn(|| { - attempt.send(()).unwrap(); - view.with_new_generation(0, |_| { - reconcile_shard(&view, 0, vec![], vec![], vec![]); - }); - complete.send(()).unwrap(); - }); - attempted.recv().unwrap(); - assert!(completed - .recv_timeout(std::time::Duration::from_millis(20)) - .is_err()); - release.send(()).unwrap(); - }); - completed.recv().unwrap(); - assert_eq!( - view.spatial - .entity_cell(EntityLayer::WorldEntity, CREATURE_BASE), - None - ); - assert!(view.auras.on_target(0, CREATURE_BASE).is_empty()); -} - #[test] fn distant_characters_do_not_add_jobs_to_a_local_combat_event() { let mut rng = Rng::new(810); From fec63d17fbe3acad6e8aa7cc9237f2e35a9cf13a Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Sat, 5 Sep 2026 00:01:08 +0200 Subject: [PATCH 7/9] fix(gateway): arm reconnect relays before resuming fresh pump --- gateway/src/stdb/connection.rs | 213 +++++++++++++++++++++++++++++---- 1 file changed, 187 insertions(+), 26 deletions(-) diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index 3cf3ef2ee..41427c920 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -60,24 +60,42 @@ pub(crate) struct LiveConn { pub(crate) reducer_completion: Arc, /// Keeps the SDK message-pump thread alive for the connection's lifetime. _pump: Option>, - pump_work: tokio::sync::mpsc::UnboundedSender, + pump_commands: tokio::sync::mpsc::UnboundedSender, /// Keeps this role's subscription active for the connection's lifetime. _sub: SubscriptionHandle, } type PumpWork = Box; +enum PumpCommand { + Run(PumpWork), + Park(std::sync::mpsc::Sender>), + Resume, + Stop, +} + /// Run setup and SDK advancement on the same thread. SDK advancement is the external edge; /// its cache update and row callbacks complete in one poll after the incoming message arrives. async fn pump_messages>>( conn: &C, - mut work: tokio::sync::mpsc::UnboundedReceiver>, + mut commands: tokio::sync::mpsc::UnboundedReceiver>, advance: impl Fn() -> F, ) -> std::result::Result<(), E> { + let mut parked = false; loop { tokio::select! { - Some(work) = work.recv() => work(conn), - result = advance() => result?, + // SubscribeApplied queues Park before returning; consume it before another SDK message. + biased; + command = commands.recv() => match command { + Some(PumpCommand::Run(work)) => work(conn), + Some(PumpCommand::Park(applied)) => { + parked = true; + let _ = applied.send(Ok(())); + } + Some(PumpCommand::Resume) => parked = false, + Some(PumpCommand::Stop) | None => return Ok(()), + }, + result = advance(), if !parked => result?, } } } @@ -90,12 +108,12 @@ fn start_pump( role: &str, ) -> Result<( std::thread::JoinHandle<()>, - tokio::sync::mpsc::UnboundedSender, + tokio::sync::mpsc::UnboundedSender, )> { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; - let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); let conn = conn.clone(); let role = role.to_owned(); let thread = std::thread::Builder::new() @@ -154,22 +172,28 @@ fn replace_live_conn( role_label: &str, post_install: impl FnOnce(), ) { + let fresh_commands = fresh.pump_commands.clone(); replace_slot_ordered( slot, fresh, role_label, |slot| { - let (conn, pump) = { + let (conn, pump, commands) = { let mut live = slot .write() .unwrap_or_else(|poisoned| poisoned.into_inner()); - (live.conn.clone(), live._pump.take()) + ( + live.conn.clone(), + live._pump.take(), + live.pump_commands.clone(), + ) }; if let Err(error) = conn.disconnect() { if !matches!(error, spacetimedb_sdk::Error::Disconnected) { log::warn!("{role_label} old connection disconnect failed: {error}"); } } + let _ = commands.send(PumpCommand::Resume); if let Some(pump) = pump { if let Err(error) = pump.join() { log::warn!("{role_label} old pump thread panicked: {error:?}"); @@ -179,6 +203,7 @@ fn replace_live_conn( || { log::info!("{role_label} replacement installed"); post_install(); + let _ = fresh_commands.send(PumpCommand::Resume); }, drop, ); @@ -412,12 +437,12 @@ impl CoordinatorInner { /// Complete Relay setup on the current connection's pump, without retaining its slot lock. pub(crate) fn on_pump(&self, work: impl FnOnce(&DbConnection) + Send + 'static) -> Result<()> { - let pump = self.coord().pump_work.clone(); + let pump = self.coord().pump_commands.clone(); let (tx, rx) = std::sync::mpsc::channel(); - pump.send(Box::new(move |conn| { + pump.send(PumpCommand::Run(Box::new(move |conn| { let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| work(conn))); let _ = tx.send(outcome.map_err(|_| "Relay setup panicked")); - })) + }))) .map_err(|_| anyhow!("{} Coordinator pump is stopped", self.db_name))?; rx.recv_timeout(Duration::from_secs(15)) .map_err(|error| anyhow!("{} Relay setup did not complete: {error}", self.db_name))? @@ -491,7 +516,7 @@ fn call_pipe_role_label(db_name: &str, index: usize) -> String { } /// Build one SDK connection generation, start its pump, and wait for the caller's exact -/// subscription policy to apply. +/// subscription policy to apply. Reconnect Coordinators park at apply until their Relays are armed. fn connect_subscribed( uri: String, db_name: String, @@ -499,6 +524,7 @@ fn connect_subscribed( queries: Vec<&'static str>, role_label: String, subscription_timeout_hint: Option<&'static str>, + park_for_relays: bool, ) -> Result { let reducer_completion = Arc::new(ReducerCompletion::connected()); let disconnected = reducer_completion.clone(); @@ -523,13 +549,18 @@ fn connect_subscribed( .map_err(|error| anyhow!("{role_label} build/connect failed: {error}"))?; let conn = Arc::new(conn); - let (pump, pump_work) = start_pump(&conn, &role_label)?; + let (pump, pump_commands) = start_pump(&conn, &role_label)?; let (tx, rx) = std::sync::mpsc::channel::>(); let tx_err = tx.clone(); + let applied_commands = pump_commands.clone(); let sub = conn .subscription_builder() .on_applied(move |_ctx| { - let _ = tx.send(Ok(())); + if park_for_relays { + let _ = applied_commands.send(PumpCommand::Park(tx)); + } else { + let _ = tx.send(Ok(())); + } }) .on_error(move |_ctx, err| { let _ = tx_err.send(Err(format!("{err}"))); @@ -550,6 +581,8 @@ fn connect_subscribed( }; if let Err(error) = applied { let _ = conn.disconnect(); + // A late apply can race this timeout and park the unpublished pump. Stop works in either state. + let _ = pump_commands.send(PumpCommand::Stop); if let Err(panic) = pump.join() { log::warn!("{role_label} failed subscription pump panicked: {panic:?}"); } @@ -561,7 +594,7 @@ fn connect_subscribed( conn, reducer_completion, _pump: Some(pump), - pump_work, + pump_commands, _sub: sub, }) } @@ -574,7 +607,15 @@ fn connect_call_pipe( index: usize, ) -> Result { let role_label = call_pipe_role_label(&db_name, index); - connect_subscribed(uri, db_name, token, call_pipe_queries(), role_label, None) + connect_subscribed( + uri, + db_name, + token, + call_pipe_queries(), + role_label, + None, + false, + ) } /// Start at most one reconnect for each failed call pipe. The coordinator watchdog owns this @@ -982,6 +1023,7 @@ fn connect_blocking( db_name: String, token: Option, sharded_tables: bool, + park_for_relays: bool, ) -> Result { let queries = coordinator_queries(sharded_tables); let role_label = coordinator_role_label(&db_name); @@ -992,6 +1034,7 @@ fn connect_blocking( queries, role_label, Some("node down, or token lacks access to the private game_account/game_session tables?"), + park_for_relays, ) } @@ -1000,7 +1043,7 @@ const COORDINATOR_WATCHDOG_POLL: Duration = Duration::from_secs(3); /// Rebuild a Coordinator when its transport, subscription, or pump stops. A Module migration /// can invalidate the subscription while the socket remains connected. -/// Build and subscribe off-lock, drain the old pump, then publish and arm the fresh connection. +/// Subscribe and park off-lock, drain the old pump, then publish, arm, and resume the fresh connection. /// Readers retain the old cache during recovery; failed connections retry on the next poll. fn spawn_coordinator_watchdog(inner: Arc) -> std::thread::JoinHandle<()> { std::thread::Builder::new() @@ -1026,6 +1069,7 @@ fn spawn_coordinator_watchdog(inner: Arc) -> std::thread::Join inner.db_name.clone(), inner.token.clone(), inner.sharded_tables, + true, ) { Ok(fresh) => { // Re-arm any relay with no per-login point to self-heal through — MUST run @@ -1600,9 +1644,12 @@ mod live_replacement_tests { #[cfg(test)] mod pump_tests { - use super::{pump_messages, PumpWork}; - use std::sync::atomic::{AtomicBool, Ordering}; + use super::{pump_messages, replace_slot_ordered, PumpCommand}; + use std::cell::RefCell; + use std::future::Future; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Mutex; + use std::task::{Context, Poll, Waker}; use std::time::Duration; use tokio::sync::mpsc; @@ -1616,12 +1663,12 @@ mod pump_tests { #[tokio::test] async fn a_quiet_pump_runs_setup_without_waiting_for_an_incoming_message() { - let (tx, rx) = mpsc::unbounded_channel::>(); + let (tx, rx) = mpsc::unbounded_channel::>(); let cache = ExternalCache::default(); let (done, finished) = tokio::sync::oneshot::channel(); - tx.send(Box::new(move |_| { + tx.send(PumpCommand::Run(Box::new(move |_| { let _ = done.send(()); - })) + }))) .unwrap(); let pumping = pump_messages(&cache, rx, std::future::pending::>); tokio::pin!(pumping); @@ -1638,17 +1685,17 @@ mod pump_tests { for (snapshot, delta) in [(None, Some(2)), (Some(1), Some(2)), (Some(1), None)] { let cache = ExternalCache::default(); *cache.row.lock().unwrap() = snapshot; - let (setup, work) = mpsc::unbounded_channel::>(); + let (setup, work) = mpsc::unbounded_channel::>(); let (incoming, messages) = mpsc::unbounded_channel::, ()>>(); let messages = tokio::sync::Mutex::new(messages); setup - .send(Box::new(move |cache| { + .send(PumpCommand::Run(Box::new(move |cache| { cache.pending_registration.store(true, Ordering::Relaxed); let snapshot = *cache.row.lock().unwrap(); incoming.send(Ok(delta)).unwrap(); incoming.send(Err(())).unwrap(); *cache.projected.lock().unwrap() = snapshot; - })) + }))) .unwrap(); let result = pump_messages(&cache, work, || async { @@ -1672,6 +1719,120 @@ mod pump_tests { assert_eq!(*cache.projected.lock().unwrap(), delta); } } + + #[tokio::test] + async fn fresh_deltas_during_retirement_wait_for_registered_relays() { + let cache = ExternalCache::default(); + let relayed = AtomicUsize::new(0); + let (commands, work) = mpsc::unbounded_channel(); + let (incoming, messages) = mpsc::unbounded_channel::>(); + let messages = tokio::sync::Mutex::new(messages); + let (applied, parked) = std::sync::mpsc::channel(); + let applied = Mutex::new(Some(applied)); + incoming.send(None).unwrap(); + let pump = RefCell::new(Box::pin(pump_messages(&cache, work, || async { + if cache.pending_registration.swap(false, Ordering::Relaxed) { + cache.registered.store(true, Ordering::Relaxed); + return Ok::<(), ()>(()); + } + match messages.lock().await.recv().await.unwrap() { + None => { + *cache.row.lock().unwrap() = Some(1); + commands + .send(PumpCommand::Park(applied.lock().unwrap().take().unwrap())) + .unwrap(); + } + Some(row) => { + *cache.row.lock().unwrap() = Some(row); + if cache.registered.load(Ordering::Relaxed) { + *cache.projected.lock().unwrap() = Some(row); + relayed.fetch_add(1, Ordering::Relaxed); + } + } + } + Ok(()) + }))); + let poll = || { + pump.borrow_mut() + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + }; + assert_eq!(poll(), Poll::Pending); + assert_eq!(parked.try_recv().unwrap(), Ok(())); + + let slot = std::sync::RwLock::new("old"); + replace_slot_ordered( + &slot, + "fresh", + "source Shard", + |_| { + incoming.send(Some(2)).unwrap(); + assert_eq!(poll(), Poll::Pending); + assert_eq!(*cache.row.lock().unwrap(), Some(1)); + assert_eq!(relayed.load(Ordering::Relaxed), 0); + }, + || { + assert_eq!(*slot.read().unwrap(), "fresh"); + commands + .send(PumpCommand::Run(Box::new(|cache| { + cache.pending_registration.store(true, Ordering::Relaxed); + }))) + .unwrap(); + assert_eq!(poll(), Poll::Pending); + assert_eq!(*cache.row.lock().unwrap(), Some(1)); + assert_eq!(relayed.load(Ordering::Relaxed), 0); + commands.send(PumpCommand::Resume).unwrap(); + assert_eq!(poll(), Poll::Pending); + }, + drop, + ); + + assert_eq!(*cache.row.lock().unwrap(), Some(2)); + assert_eq!(*cache.projected.lock().unwrap(), Some(2)); + assert_eq!(relayed.load(Ordering::Relaxed), 1); + commands.send(PumpCommand::Stop).unwrap(); + assert_eq!(poll(), Poll::Ready(Ok(()))); + } + + #[tokio::test] + async fn a_quiet_parked_pump_resumes_without_an_incoming_message() { + let (commands, work) = mpsc::unbounded_channel(); + let (applied, parked) = std::sync::mpsc::channel(); + commands.send(PumpCommand::Park(applied)).unwrap(); + let advances = AtomicUsize::new(0); + let mut pump = Box::pin(pump_messages(&(), work, || async { + advances.fetch_add(1, Ordering::Relaxed); + std::future::pending::>().await + })); + let mut cx = Context::from_waker(Waker::noop()); + assert_eq!(pump.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(parked.try_recv().unwrap(), Ok(())); + assert_eq!(advances.load(Ordering::Relaxed), 0); + commands.send(PumpCommand::Resume).unwrap(); + assert_eq!(pump.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!(advances.load(Ordering::Relaxed), 1); + commands.send(PumpCommand::Stop).unwrap(); + assert_eq!(pump.as_mut().poll(&mut cx), Poll::Ready(Ok(()))); + } + + #[tokio::test] + async fn failed_subscription_cleanup_stops_before_or_after_a_late_apply() { + for stop_first in [true, false] { + let (commands, work) = mpsc::unbounded_channel(); + let (applied, _parked) = std::sync::mpsc::channel(); + if stop_first { + commands.send(PumpCommand::Stop).unwrap(); + commands.send(PumpCommand::Park(applied)).unwrap(); + } else { + commands.send(PumpCommand::Park(applied)).unwrap(); + commands.send(PumpCommand::Stop).unwrap(); + } + assert_eq!( + pump_messages(&(), work, std::future::pending::>).await, + Ok(()) + ); + } + } } #[cfg(test)] @@ -1914,7 +2075,7 @@ impl Coordinator { // Build on a dedicated OS thread (no tokio context), join it off the reactor. let build_thread = std::thread::Builder::new() .name("stdb-coordinator-connect".into()) - .spawn(move || connect_blocking(uri, db, token, sharded_tables)) + .spawn(move || connect_blocking(uri, db, token, sharded_tables, false)) .context(format!("spawn {role_label} connect thread"))?; let live = tokio::task::spawn_blocking(move || -> Result { build_thread From 3ae1c0d9207f22b579cc43e83e8a3196dc261d00 Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Sat, 5 Sep 2026 00:12:11 +0200 Subject: [PATCH 8/9] test(gateway): clarify relay routing and require melee audience pruning --- docs/architecture.md | 8 ++++---- gateway/src/stdb/world_index.rs | 4 ++-- gateway/src/stdb/world_view.rs | 6 ++++-- gateway/src/stdb/world_view/cell_audience_tests.rs | 6 ++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index cc1e57e71..47722fed3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -358,10 +358,10 @@ Every relay hangs off a coordinator connection, in one of two shapes: families (entities, motion, combat, chat, auras, corpses, casts), the recipient-keyed PRIVATE tier (whisper/group/resurrect), and owner-addressed XP, level-up, exploration, quest, item, teleport, addon, and reputation rows. GUID and bound-identity indexes select one viewer directly; - the callback enqueues packet work on that session's FIFO writer. Combat, melee stance, cast, - impact, emote, chat and aura rows are cell-anchored: the Module stamps the actor's cell on the - event row (an aura or chat line uses the actor's indexed cell), the shared cell index picks the - viewers whose box covers it, and the job's per-viewer gate stays the final filter. Only rolls, + the callback enqueues packet work on that session's FIFO writer. Combat, cast, impact, and emote + rows carry the actor's cell. Melee stance uses the attacker's indexed cell, chat uses the sender's, + and auras use the target's. The shared cell index selects nearby viewers and named owners on the + source Shard, and the job's per-viewer gate stays the final filter. Only rolls, corpses, dynamic objects, channel lines and weather still fan out per shard. The cross-shard whisper/group twins ride the same dispatchers on the realm-core connection (`arm_realm_private`), armed only when realm-core is a distinct database. diff --git a/gateway/src/stdb/world_index.rs b/gateway/src/stdb/world_index.rs index 624ade461..619434be8 100644 --- a/gateway/src/stdb/world_index.rs +++ b/gateway/src/stdb/world_index.rs @@ -559,7 +559,7 @@ impl WorldIndex { (ax - cx).abs() <= BOX_HALF_SPAN && (ay - cy).abs() <= BOX_HALF_SPAN } - /// Live sizes, for the periodic ops log line: `(entities, viewers, occupied entity cells)`. + /// Index sizes for tests: `(entities, viewers, occupied entity cells)`. #[cfg(test)] pub fn stats(&self, layer: EntityLayer) -> (usize, usize, usize) { let inner = self.lock(); @@ -1060,7 +1060,7 @@ mod tests { } #[test] - fn stats_report_what_the_ops_line_prints() { + fn stats_count_entities_viewers_and_occupied_cells() { let index = WorldIndex::new(true); index.upsert_entity(EntityLayer::WorldEntity, 1, CellKey::at(0, 0, 0, 0), 0, 0); index.upsert_entity(EntityLayer::WorldEntity, 2, CellKey::at(0, 0, 0, 0), 0, 0); diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index 4f17f27de..755cbc086 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -80,7 +80,7 @@ pub(crate) struct Viewer { /// weather for, so the first stored value is never a phantom crossing. pub(crate) zone_id: AtomicU32, pub(crate) tx: SessionTx, - /// The per-viewer "already shown" dedup set — the exactly-once guarantee behind every CREATE. + /// WorldEntity GUIDs already shown to this viewer. pub(crate) created: Arc>>, pub(crate) gates: Arc, /// PLAYER_SKILL_INFO slot map `(skill_line → slot, next_free)` — seeded at login from the same @@ -3611,7 +3611,7 @@ fn reconcile_shard( } } } - // Existing viewers keep their CREATE dedup state. Newly resident rows are offered normally. + // Existing WorldEntity rows retain their CREATE dedup state. for row in entities { if view.spatial.shard_of(EntityLayer::WorldEntity, row.guid) != Some(shard) { continue; @@ -3627,6 +3627,8 @@ fn reconcile_shard( offer_create_job(view, shard, session, &row); } } + // Refresh resident GameObjects too: state and rotation may have changed while disconnected. + // Their CREATE packets do not use the WorldEntity dedup set. for row in objects { if view.spatial.shard_of(EntityLayer::GameObject, row.guid) == Some(shard) { gameobject_appeared(view, shard, &row); diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index 9b3924af7..e1784e796 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -365,6 +365,7 @@ fn combat_candidates_gate_to_exactly_the_whole_shard_recipients() { #[test] fn melee_candidates_gate_to_exactly_the_whole_shard_recipients() { let mut rng = Rng::new(0x3E1E_E002); + let mut skipped_someone = false; for _ in 0..200 { let realm = spread_realm(&mut rng); let attacker = realm.actor(&mut rng); @@ -384,7 +385,12 @@ fn melee_candidates_gate_to_exactly_the_whole_shard_recipients() { .map(|v| v.session) .collect(); assert_eq!(gated, realm.shard_then_gate(gate)); + skipped_someone |= candidates.len() < realm.viewers.len(); } + assert!( + skipped_someone, + "a spread realm must leave some viewers out of the box" + ); } /// An aura job passes its gate for the target itself, for a viewer showing the target, or (the From d1b34d494706d8f4c83afa2da02767ec660f4440 Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Sat, 5 Sep 2026 00:28:11 +0200 Subject: [PATCH 9/9] fix(gateway): fence local relays and stop incomplete reconnects --- gateway/src/stdb/connection.rs | 53 ++++++++++- gateway/src/stdb/world_view.rs | 92 ++++++------------- .../stdb/world_view/cell_audience_tests.rs | 86 +++++++++++++++++ 3 files changed, 163 insertions(+), 68 deletions(-) diff --git a/gateway/src/stdb/connection.rs b/gateway/src/stdb/connection.rs index 41427c920..07c1d26a2 100644 --- a/gateway/src/stdb/connection.rs +++ b/gateway/src/stdb/connection.rs @@ -163,6 +163,33 @@ fn replace_slot_ordered( teardown(old); } +/// An unwinding hook must stop the fresh pump before partially installed Relays can run. +fn finish_relay_setup( + commands: &tokio::sync::mpsc::UnboundedSender>, + setup: impl FnOnce(), +) { + struct Completion<'a, C> { + commands: &'a tokio::sync::mpsc::UnboundedSender>, + complete: bool, + } + impl Drop for Completion<'_, C> { + fn drop(&mut self) { + let command = if self.complete { + PumpCommand::Resume + } else { + PumpCommand::Stop + }; + let _ = self.commands.send(command); + } + } + let mut completion = Completion { + commands, + complete: false, + }; + setup(); + completion.complete = true; +} + /// Drain the old pump before exposing the fresh cache. A fresh source cache can advance a /// Transfer to its destination, so no old source callback may update the shared indexes afterward. /// The slot lock is released before disconnect or join; callbacks can finish their cache reads. @@ -202,8 +229,7 @@ fn replace_live_conn( }, || { log::info!("{role_label} replacement installed"); - post_install(); - let _ = fresh_commands.send(PumpCommand::Resume); + finish_relay_setup(&fresh_commands, post_install); }, drop, ); @@ -1644,7 +1670,7 @@ mod live_replacement_tests { #[cfg(test)] mod pump_tests { - use super::{pump_messages, replace_slot_ordered, PumpCommand}; + use super::{finish_relay_setup, pump_messages, replace_slot_ordered, PumpCommand}; use std::cell::RefCell; use std::future::Future; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -1808,13 +1834,32 @@ mod pump_tests { assert_eq!(pump.as_mut().poll(&mut cx), Poll::Pending); assert_eq!(parked.try_recv().unwrap(), Ok(())); assert_eq!(advances.load(Ordering::Relaxed), 0); - commands.send(PumpCommand::Resume).unwrap(); + finish_relay_setup(&commands, || {}); assert_eq!(pump.as_mut().poll(&mut cx), Poll::Pending); assert_eq!(advances.load(Ordering::Relaxed), 1); commands.send(PumpCommand::Stop).unwrap(); assert_eq!(pump.as_mut().poll(&mut cx), Poll::Ready(Ok(()))); } + #[tokio::test] + async fn a_panicking_reconnect_hook_stops_its_partially_armed_pump() { + let (commands, work) = mpsc::unbounded_channel(); + let (applied, _parked) = std::sync::mpsc::channel(); + commands.send(PumpCommand::Park(applied)).unwrap(); + let mut pump = Box::pin(pump_messages( + &(), + work, + std::future::pending::>, + )); + let mut cx = Context::from_waker(Waker::noop()); + assert_eq!(pump.as_mut().poll(&mut cx), Poll::Pending); + let outcome = std::panic::catch_unwind(|| { + finish_relay_setup(&commands, || panic!("hook failed")); + }); + assert!(outcome.is_err(), "hook failures still propagate"); + assert_eq!(pump.as_mut().poll(&mut cx), Poll::Ready(Ok(()))); + } + #[tokio::test] async fn failed_subscription_cleanup_stops_before_or_after_a_late_apply() { for stop_first in [true, false] { diff --git a/gateway/src/stdb/world_view.rs b/gateway/src/stdb/world_view.rs index 755cbc086..87fbce1b4 100644 --- a/gateway/src/stdb/world_view.rs +++ b/gateway/src/stdb/world_view.rs @@ -767,7 +767,7 @@ fn register_shard_callbacks( db.game_rest_state_event(), "game_rest_state_event.insert", &view, - |v, row| rest_state_appeared(v, row), + move |v, row| rest_state_appeared(v, shard, row), ); // ---- game_breath_relay_event ------------------------------------------------------------ @@ -776,7 +776,7 @@ fn register_shard_callbacks( db.game_breath_relay_event(), "game_breath_relay_event.insert", &view, - |v, row| breath_relay_appeared(v, row), + move |v, row| breath_relay_appeared(v, shard, row), ); // ---- game_dynamic_object ----------------------------------------------------------------- @@ -853,7 +853,7 @@ fn register_shard_callbacks( db.game_resurrect_request(), "game_resurrect_request.insert", &view, - |v, row| resurrect_offered(v, row), + move |v, row| resurrect_offered(v, shard, row), ); wire_insert_live( db.game_whisper_event(), @@ -882,21 +882,21 @@ fn register_shard_callbacks( db.game_group_member(), "game_group_member.insert", &view, - move |v, row| group_member_mirror_changed(v, &insert_coord, row), + move |v, row| group_member_mirror_changed(v, &insert_coord, shard, row), ); let delete_coord = coord.clone(); wire_delete( db.game_group_member(), "game_group_member.delete", &view, - move |v, row| group_member_mirror_changed(v, &delete_coord, row), + move |v, row| group_member_mirror_changed(v, &delete_coord, shard, row), ); } wire_insert_live( db.game_trade_event(), "game_trade_event.insert", &view, - |v, row| trade_event_appeared(v, row), + move |v, row| trade_event_appeared(v, shard, row), ); { let coord = coord.clone(); @@ -904,7 +904,7 @@ fn register_shard_callbacks( db.game_duel_event(), "game_duel_event.insert", &view, - move |v, row| duel_event_appeared(v, &coord, row), + move |v, row| duel_event_appeared(v, &coord, shard, row), ); } @@ -1757,19 +1757,9 @@ fn item_deleted(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &Ite }); } -/// A rest-state flip landed on a shard's coordinator feed. Self-only family: the row names its -/// owner and the owner is the ONLY lawful recipient, so the audience predicate IS the -/// owner-session lookup — there is no candidate set to filter, -/// and a row whose owner has no live session here is dropped. -/// -/// This family needs no replay state: nothing re-subscribes per session on this path, so rows -/// arriving after arming are live flips. A coordinator watchdog re-arm can replay rows still -/// inside their GC TTL, which idempotently re-sends the current rest byte. -fn rest_state_appeared(view: &WorldView, row: &RestStateEvent) { - let Some(session) = view.session_of_owner(row.character_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +/// Send a rest-state change to its owner on the callback's Shard. +fn rest_state_appeared(view: &WorldView, shard: ShardId, row: &RestStateEvent) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; let (self_guid, bytes) = (row.character_guid, row.player_bytes_2); @@ -1778,12 +1768,9 @@ fn rest_state_appeared(view: &WorldView, row: &RestStateEvent) { }); } -/// Breath relay events are delivered only to their owner. -fn breath_relay_appeared(view: &WorldView, row: &BreathRelayEvent) { - let Some(session) = view.session_of_owner(row.character_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +/// Send breath events to their owner on the callback's Shard. +fn breath_relay_appeared(view: &WorldView, shard: ShardId, row: &BreathRelayEvent) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; let (guid, kind, remaining, duration, damage) = ( @@ -1930,11 +1917,8 @@ fn melee_disengaged(view: &WorldView, shard: ShardId, row: &MeleeAttack) { } /// A resurrect offer landed → SMSG_RESURRECT_REQUEST to the offer's TARGET and nobody else. -fn resurrect_offered(view: &WorldView, row: &ResurrectRequest) { - let Some(session) = view.session_of_owner(row.target_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +fn resurrect_offered(view: &WorldView, shard: ShardId, row: &ResurrectRequest) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.target_guid)) else { return; }; if !super::subscriptions::private_recipient_audience(row.target_guid, viewer.self_guid) { @@ -1984,11 +1968,8 @@ fn system_message_appeared(view: &WorldView, row: &SystemMessageEvent) { /// A trade status landed → the `SMSG_TRADE_STATUS` packet to the row's RECIPIENT and nobody /// else (same shape as [`whisper_appeared`]) (#120). -fn trade_event_appeared(view: &WorldView, row: &TradeEvent) { - let Some(session) = view.session_of_owner(row.recipient_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +fn trade_event_appeared(view: &WorldView, shard: ShardId, row: &TradeEvent) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.recipient_guid)) else { return; }; if !super::subscriptions::private_recipient_audience(row.recipient_guid, viewer.self_guid) { @@ -2002,10 +1983,9 @@ fn trade_event_appeared(view: &WorldView, row: &TradeEvent) { /// A Duel lifecycle edge landed for one participant. Private packets go only to that participant; /// the initiator-addressed completion row also drives the one nearby winner announcement. -fn duel_event_appeared(view: &WorldView, coord: &Coordinator, row: &DuelEvent) { +fn duel_event_appeared(view: &WorldView, coord: &Coordinator, shard: ShardId, row: &DuelEvent) { if let Some(viewer) = view - .session_of_owner(row.recipient_guid) - .and_then(|session| view.viewer(session)) + .viewer_of_owner_on_shard(shard, OwnerGuid(row.recipient_guid)) .filter(|viewer| { super::subscriptions::private_recipient_audience(row.recipient_guid, viewer.self_guid) }) @@ -2030,10 +2010,7 @@ fn duel_event_appeared(view: &WorldView, coord: &Coordinator, row: &DuelEvent) { } let key = CellKey::of_position(row.map_id, row.instance_id, row.flag_x, row.flag_y); let row = Arc::new(row.clone()); - for session in view.spatial.viewers_of(EntityLayer::WorldEntity, key) { - let Some(viewer) = view.viewer(session) else { - continue; - }; + for viewer in view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[]) { if !duel_winner_audience(viewer.self_guid, row.initiator_guid, row.challenged_guid) { continue; } @@ -2066,11 +2043,13 @@ fn group_event_appeared(view: &WorldView, coord: &Coordinator, row: &GroupEvent) /// A shard mirror delta is the point at which viewer-relative Loot Tag flags can be projected from /// current membership without racing the realm event that requested the mirror write. -fn group_member_mirror_changed(view: &WorldView, coord: &Coordinator, row: &GroupMember) { - let Some(session) = view.session_of_owner(row.character_guid) else { - return; - }; - let Some(viewer) = view.viewer(session) else { +fn group_member_mirror_changed( + view: &WorldView, + coord: &Coordinator, + shard: ShardId, + row: &GroupMember, +) { + let Some(viewer) = view.viewer_of_owner_on_shard(shard, OwnerGuid(row.character_guid)) else { return; }; let coord = coord.clone(); @@ -2713,25 +2692,10 @@ mod family_audience_tests { let source = include_str!("world_view.rs"); let body = crate::test_scan::code_of(source, "fn duel_event_appeared("); assert!(body.contains("CellKey::of_position")); - assert!(body.contains("view.spatial.viewers_of(EntityLayer::WorldEntity, key)")); + assert!(body.contains("view.cell_audience(shard, Some(key), BOX_HALF_SPAN, &[])")); assert!(body.contains("duel_winner_outbound")); } - #[test] - fn breath_relay_owner_lookup_selects_only_its_own_session() { - let view = WorldView::new(true); - // No owner session on this gateway: the relay must be dropped. - assert_eq!(view.session_of_owner(41), None); - // The registry tests pin the registration mechanics; the callback source scan below - // pins that breath events actually take this exact lookup rather than an AOI broadcast. - let source = include_str!("world_view.rs"); - let start = source.find("fn breath_relay_appeared").unwrap(); - let body = &source[start..]; - assert!(body.contains("view.session_of_owner(row.character_guid)")); - assert!(!body[..body.find("fn skill_changed").unwrap()] - .contains("viewers_on_shard"), "breath rows must never broadcast to shard viewers"); - } - #[test] fn one_recenter_reports_both_layers_and_keeps_equal_guids_independent() { let view = WorldView::new(true); diff --git a/gateway/src/stdb/world_view/cell_audience_tests.rs b/gateway/src/stdb/world_view/cell_audience_tests.rs index e1784e796..ba0743b59 100644 --- a/gateway/src/stdb/world_view/cell_audience_tests.rs +++ b/gateway/src/stdb/world_view/cell_audience_tests.rs @@ -862,6 +862,92 @@ fn jobs_selected_before_worldport_cannot_reach_the_replacement_world_session() { )); } +#[test] +fn local_owner_events_cannot_reach_a_destination_world_session() { + use wow_world_messages::vanilla::opcodes::ServerOpcodeMessage; + let view = WorldView::new(true); + let cell = CellKey::at(0, 0, 0, 0); + let (old_tx, _old_rx) = SessionTx::with_depth(0); + view.add_viewer_on_shard(viewer(1, PLAYER_BASE, old_tx), cell, 0); + view.remove_viewer(1); + let (tx, rx) = SessionTx::with_depth(0); + view.add_viewer_on_shard(viewer(2, PLAYER_BASE, tx), cell, 1); + let (other_tx, other_rx) = SessionTx::with_depth(0); + view.add_viewer_on_shard(viewer(3, PLAYER_BASE + 1, other_tx), cell, 1); + let rest = RestStateEvent { + id: 1, + character_guid: PLAYER_BASE, + player_bytes_2: 0x0100_0000, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + }; + let breath = BreathRelayEvent { + id: 2, + character_guid: PLAYER_BASE, + kind: 0, + time_remaining_ms: 12000, + duration_ms: 60000, + damage: 0, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + }; + let resurrect = ResurrectRequest { + target_guid: PLAYER_BASE, + target_identity: identity(PLAYER_BASE), + caster_guid: PLAYER_BASE + 1, + caster_name: "Caster".into(), + points: 50, + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + }; + let trade = TradeEvent { + id: 3, + recipient_identity: identity(PLAYER_BASE), + kind: lyracore_shared::trade::event_kind::TRADE_CANCELED, + other_guid: PLAYER_BASE + 1, + recipient_guid: PLAYER_BASE, + payload: String::new(), + created_at: spacetimedb_sdk::Timestamp::UNIX_EPOCH, + }; + for shard in [0, 1] { + rest_state_appeared(&view, shard, &rest); + breath_relay_appeared(&view, shard, &breath); + resurrect_offered(&view, shard, &resurrect); + trade_event_appeared(&view, shard, &trade); + assert!( + other_rx.try_recv().is_err(), + "owner rows must not reach a bystander" + ); + if shard == 0 { + assert!( + rx.try_recv().is_err(), + "late source rows must not reach the destination" + ); + } + } + let packets: Vec<_> = rx + .try_iter() + .flat_map(|outbound| { + let Outbound::Job(job) = outbound else { + panic!("expected a Relay job") + }; + job() + }) + .collect(); + assert_eq!(packets.len(), 4); + assert!(matches!(&packets[0], Outbound::Raw { opcode: 0xA9, body } + if body.ends_with(&0x0100_0000_u32.to_le_bytes()))); + assert!(matches!( + &packets[1], + Outbound::One(ServerOpcodeMessage::SMSG_START_MIRROR_TIMER(_)) + )); + assert!(matches!( + &packets[2], + Outbound::One(ServerOpcodeMessage::SMSG_RESURRECT_REQUEST(_)) + )); + assert!( + matches!(&packets[3], Outbound::One(ServerOpcodeMessage::SMSG_TRADE_STATUS(status)) + if matches!(status.as_ref(), wow_world_messages::vanilla::SMSG_TRADE_STATUS::TradeCanceled)) + ); +} + #[test] fn spatial_candidates_also_stay_on_the_event_shard() { let view = WorldView::new(true);