feat(importer): load all creature definitions and map populations

CONTEXT.md

@@ -70,9 +70,13 @@ The shard that hosts instanced maps.
 **World Import Profile**:
 A stable name for the import plan assigned to one World Shard or Instance Pool destination. The
 canonical profiles are `alliance-eastern`, `alliance-kalimdor`, `alliance-single`,
-`starting-eastern`, `starting-kalimdor`, and `instances`. The starting profiles retain the Alliance
-scope and add the Horde starting areas on the same continent. Canonical profiles share quest
-definitions and scripted event requirements from both starting profiles and the Instance Pool.
+`starting-eastern`, `starting-kalimdor`, `instances`, `full-world`, `full-eastern`,
+`full-kalimdor`, and `full-instances`. Full profiles share every creature definition and build a shared quest catalogue from the
+full source. Their spawn positions follow the selected maps; `full-world` accepts
+every source map. Partitioned full profiles compile AI for local populations and their summoned
+creatures. The starting profiles retain the Alliance scope and add the Horde starting areas on
+the same continent. Smaller canonical profiles share quest definitions and scripted event
+requirements from both starting profiles and the Instance Pool.
 Quest-giver relations stay within each destination's World Import Scope.
 
 **World Import Scope**:

crates/lyracore-shared/src/chat.rs

@@ -46,6 +46,14 @@ pub mod broadcast_chat {
     pub const CREATURE_TEXT_EMOTE: u8 = 2;
     /// A Character's `/e` custom emote (cm:ChatHandler.cpp:172-226, cm:Player.cpp:16591-16599).
     pub const EMOTE: u8 = 3;
+    pub const CREATURE_BOSS_EMOTE: u8 = 4;
+    pub const CREATURE_WHISPER: u8 = 5;
+    pub const CREATURE_ZONE_YELL: u8 = 6;
+    pub const CREATURE_ZONE_EMOTE: u8 = 7;
+
+    pub fn is_zone_wide(chat_type: u8) -> bool {
+        matches!(chat_type, CREATURE_ZONE_YELL | CREATURE_ZONE_EMOTE)
+    }
 
     /// `/e` and text-emote (`/wave`, `/dance`, …) proximity range: `ListenRange.TextEmote`
     /// (cm:mangosd.conf.dist.in:1092), same as `ListenRange.Say`.

crates/lyracore-shared/src/instance.rs

@@ -1,19 +1,25 @@
-//! Shared dungeon-map classification and Instance Removal countdown.
-//! The Module uses the map set to create dungeon Instances. The Gateway uses it to verify
-//! that each dungeon has a Shard that hosts its population.
-
-/// The maps that are DUNGEONS — entering an areatrigger portal targeting one of these resolves-or-
-/// creates an instance instead of landing at instance 0.
-///
-/// Every entry MUST also have a `module::instance::entrance_fallback` arm (unit-pinned there) so a
-/// reaped-instance login can never strand.
-pub const DUNGEON_MAPS: &[u32] = &[36];
-
-/// Is `map_id` a dungeon (instanced) map? See [`DUNGEON_MAPS`]. Pure.
+//! Shared vanilla dungeon classification and Instance Removal countdown.
+
+/// Vanilla dungeon and raid maps. Battlegrounds use a separate population lifecycle.
+/// The client Map.dbc instance types and ClassicDB instance_template identify this set.
+pub const DUNGEON_MAPS: &[u32] = &[
+    33, 34, 36, 43, 47, 48, 70, 90, 109, 129, 189, 209, 229, 230, 249, 289, 309, 329, 349, 389,
+    409, 429, 469, 509, 531, 533,
+];
+
 pub fn is_dungeon_map(map_id: u32) -> bool {
     DUNGEON_MAPS.contains(&map_id)
 }
 
+/// Source spawns on these maps remain dormant until their instance creates a population.
+/// Battleground populations stay dormant until battleground creation is implemented.
+/// Shared interiors, including the Deeprun Tram and faction halls, use instance zero.
+pub const BATTLEGROUND_MAPS: &[u32] = &[30, 489, 529];
+
+pub fn has_instance_population(map_id: u32) -> bool {
+    is_dungeon_map(map_id) || BATTLEGROUND_MAPS.contains(&map_id)
+}
+
 /// The Instance Removal countdown in milliseconds (cm:Player.cpp:17723). The Module schedules the
 /// expiry this far out, and the Gateway never shows the client more time than this.
 pub const INSTANCE_REMOVAL_MS: u32 = 60_000;
@@ -23,8 +29,18 @@ mod tests {
     use super::*;
 
     #[test]
-    fn the_dungeon_set_holds_deadmines_and_no_open_world_map() {
-        assert!(is_dungeon_map(36), "Deadmines is the one imported dungeon");
+    fn dungeons_include_vanilla_raids_but_exclude_battlegrounds_and_shared_interiors() {
+        assert!(is_dungeon_map(36), "Deadmines is instanced");
+        for map_id in [33, 229, 249, 309, 409, 469, 509, 531, 533] {
+            assert!(is_dungeon_map(map_id));
+        }
+        for map_id in [0, 1, 369, 449, 450] {
+            assert!(!has_instance_population(map_id));
+        }
+        for map_id in [30, 489, 529] {
+            assert!(!is_dungeon_map(map_id));
+            assert!(has_instance_population(map_id));
+        }
         assert!(!is_dungeon_map(0), "Eastern Kingdoms is open world");
         assert!(!is_dungeon_map(1), "Kalimdor is open world");
     }

docs/data-ingestion.md

@@ -104,13 +104,11 @@ live subscriptions already *are* the visibility/broadcast layer (a creature is "
 language (Rust) end-to-end reusing the gtker crate family (incl. `wow_dbc`); our schema is already a
 subset so growth is append-only.
 
-**Tradeoffs:** full-world volume is unproven for us (~1.4 GB MySQL as a size proxy); reducer
-arg-size/exec limits **forbid streaming the whole dump through one call** → import MUST be externally
-chunked/batched via a privileged admin/gateway SDK path; **seed-as-code (`seed.rs`) does NOT scale**
-(~10k creatures / ~66k spawns / ~17k items / ~170k loot rows would wreck compile time + bloat the
-wasm) → need a **data-bundle loader**, content versioned separately from code; migrations are
-append-only (never reshape); we'd own an ETL + validation that mangos has 15 yrs of tooling for;
-mangos column *semantics* don't always map 1:1 (e.g. multiplier-based stats vs our flat values).
+**Tradeoffs:** production capacity needs measurement on the target topology. Imports use external
+batches to stay within reducer argument and execution limits. The pinned source has 10,384
+creature definitions and 66,310 positions; expanded creature loot reaches 1,316,655 rows.
+Content loads from data instead of compiled seed code. Migrations remain append-only, and source
+semantics such as multiplier-based stats still need explicit mappings to the Module's fields.
 
 **Republish & data retention (re-verified 2026-06-17, Slice 2):** a plain `spacetime publish` (no
 `-c`) is **never destructive** — it either auto-migrates while RETAINING all data, or it *aborts*
@@ -204,18 +202,57 @@ Ship **zero** third-party-derived data. Three streams, each with its own provena
 - **P5 — Quests** (+ the npc/gossip relations they need): densest behavioral content; depends on
   creatures/items/gossip.
 
-**Per-shard scope.** Once the world is sharded by continent, the ETL's unit of work is one map into
-one database — the importer takes the target database, the map id, and a bounding box, and its map
-fence plus the surrounding pre- and post-flight guards keep any one database from holding two
-continents' spatial rows. Non-spatial content (spells, items, quests, the DBC reference tables) is
-duplicated on every shard by design.
-
-Canonical World Import Profiles build quest definitions from the union of the starting profiles
-and the Instance Pool. This keeps quest text, objectives, and scripted event requirements
-available after Transfer. Quest reimports retain summoned givers and reapply the `creature-ai`
-Package Deltas for event requirements. Quest-giver relations still follow the destination's World Import Scope. Legacy map and box imports retain
-their selected quest scope. Existing canonical imports need the `quests` and `items` families
-reimported on every World Shard and Instance Pool so the shared quest rewards have item templates.
+**Per-shard scope.** World Import Profiles assign spatial rows to a World Shard or Instance Pool.
+`full-eastern` owns maps 0, 369 and 449; `full-kalimdor` owns maps 1 and 450;
+`full-instances` owns the dungeon, raid and battleground maps. Each full profile imports every
+creature definition and builds a shared quest catalogue from the full source. Runnable AI follows local creature
+populations and their summon dependencies, so its spatial references stay on their owning Shard.
+Quest-giver relations follow local spawns.
+`full-world` imports every source map into one database. Legacy map and box imports retain their
+selected scope. Reimports replace each selected family; two runs do not combine spatial scopes.
+
+For a complete source import, supply the dump and client data together:
+
+```sh
+cargo run -p lyracore-importer -- \
+  --dump .import/classic-db-full.sql --dbc /path/to/client/Data \
+  --world-profile full-world
+```
+
+This prints the plan. Add `--apply --server <local-node> --db <database>` to load it. Full profiles
+load the client reference catalogues from the same dump and client inputs, including taxi mount
+models. They do not require a separate `--taxi-mounts` argument. The script also loads spells,
+talents, terrain and navigation:
+
+```sh
+WORLD_PROFILE=full-world DUMP=.import/classic-db-full.sql DBC=/path/to/client/Data \
+  DB=local-full-world SPACETIME_SERVER=http://127.0.0.1:3000 \
+  bash importer/scripts/import-world.sh
+```
+
+All authored creature positions are retained, including nonselectable creatures and population
+alternatives. Population metadata retains entry choices, count limits, chances and event links.
+Legacy creature pools retain fractional chances and template-based membership. Equal-sized,
+equal-chance nested pools with single-selection limits flatten to the same initial distribution.
+Other nesting and mixed creature/GameObject pools stop the import before it writes data.
+The Module selects an initial population from that metadata. Selections last until reimport or
+Instance teardown; source pool rotation after deaths is not yet implemented. Positive game-event
+members and groups with world-state conditions remain dormant until an event system can evaluate
+them. Negative event members use the inactive-event state. Conditional squad positions with no
+SQL entry choices remain inspectable source rows with entry zero and never become live entities.
+
+Dungeon, raid and battleground source rows stay dormant in Instance zero. Creating a dungeon or
+raid Instance copies its selected creatures and patrols into that Instance. The copies respawn
+and are removed with the Instance. This supplies map populations; it does not supply missing
+encounter scripts, raid lockouts or battleground rules. Continent terrain covers the full client
+grid; imported spawn coverage does not imply complete collision coverage for every interior.
+Zone-wide creature speech requires imported terrain and area metadata to resolve its audience.
+Reimport clears creature copies in existing Instances but retains their leases. Recreate those
+Instances before resuming play.
+
+The EventAI compatibility gate still applies to the full catalogue. A larger scope does not
+approve dropped rules or unknown behavior. Existing smaller profiles retain their shared starting
+quest catalogue and their local quest-giver relations.
 
 ## Creature EventAI family
 

docs/instance-lifecycle.md

@@ -19,9 +19,16 @@ after 30 minutes empty, or on the next empty observation when `reset_requested`
 clears the stamp. Teardown removes population, encounter state, the dedicated tick row, Character
 bindings, and finally the lease.
 
-Instance creature populations have no authored spawn rows of their own. Trash does not respawn,
-corpses remain until reap, and waypoint, wander, and return-home movement is unavailable. Aggro,
-assist, chase, flee, and casting use entity and template state and remain active.
+Each Instance copies its selected source spawn rows and waypoint routes. The copies own their death and
+respawn timers, movement home, and life sequence. Reaping an Instance deletes those copies,
+including dead creatures awaiting respawn. Imported dungeon and battleground source rows stay
+dormant at instance zero. Battleground creation remains unimplemented. The Deeprun Tram and faction
+halls use shared populations at instance zero.
+
+Dungeon exits and graveyard release use imported Area Trigger, area, and graveyard metadata.
+Missing exit metadata sends a reaped-instance login to the Character's home. The small fixture keeps
+its Deadmines bootstrap fallback. Raid admission uses map capacity; saved raid lockouts and encounter
+scripts remain separate gameplay work.
 
 Creature copies use `encounter::wave_guid` with bit 23 set in the low field. This separates their
 identities from imported spawns and encounter wave allocation. GameObject copies use

docs/testing.md

@@ -141,6 +141,15 @@ For example:
 LYRACORE_CLIENT_DATA=/games/WoW-1.12.1/Data cargo test -p lyracore-importer terrain -- --ignored
 ```
 
+The complete creature import test loads the pinned dump and client catalogues into a private local
+SpacetimeDB node, checks every source position, waits for the shared-world population, and reimports
+the populated roster:
+
+```bash
+LYRACORE_CLASSIC_DB_SQL=/path/to/classic.sql LYRACORE_TEST_DBC=/path/to/client/Data \
+  cargo test -p lyracore-importer full_world_source_loads_every_definition_and_spawn_locally -- --ignored
+```
+
 Run the terrain check before an `--apply` after you change a profile anchor.
 
 A Package test runner sets `LYRACORE_TEST_CORE` to the Core checkout it builds against.

gateway/src/codec/social.rs

@@ -210,20 +210,35 @@ pub fn build_chat_message_to(
             sender_name,
             target: Guid::new(target_guid),
         },
-        (1, Some(sender_name)) => SMSG_MESSAGECHAT_ChatType::MonsterYell {
-            sender1: sender,
-            sender_name,
-            target: Guid::new(target_guid),
-        },
+        (1 | broadcast_chat::CREATURE_ZONE_YELL, Some(sender_name)) => {
+            SMSG_MESSAGECHAT_ChatType::MonsterYell {
+                sender1: sender,
+                sender_name,
+                target: Guid::new(target_guid),
+            }
+        }
         // `MonsterEmote` carries one guid and it names the SPEAKER, paired with the speaker's own
         // name. The variant has no addressee field on the wire, so an emote's addressed target is
         // not expressible here and is dropped rather than misfiled into the speaker slot.
-        (broadcast_chat::CREATURE_TEXT_EMOTE, Some(monster_name)) => {
-            SMSG_MESSAGECHAT_ChatType::MonsterEmote {
+        (broadcast_chat::CREATURE_WHISPER, Some(monster_name)) => {
+            SMSG_MESSAGECHAT_ChatType::MonsterWhisper {
                 monster: sender,
                 monster_name,
             }
         }
+        (broadcast_chat::CREATURE_BOSS_EMOTE, Some(monster_name)) => {
+            SMSG_MESSAGECHAT_ChatType::RaidBossEmote {
+                monster: sender,
+                monster_name,
+            }
+        }
+        (
+            broadcast_chat::CREATURE_TEXT_EMOTE | broadcast_chat::CREATURE_ZONE_EMOTE,
+            Some(monster_name),
+        ) => SMSG_MESSAGECHAT_ChatType::MonsterEmote {
+            monster: sender,
+            monster_name,
+        },
         (1, None) => SMSG_MESSAGECHAT_ChatType::Yell {
             chat_credit: sender,
             speech_bubble_credit: sender,
@@ -243,7 +258,10 @@ pub fn build_chat_message_to(
         chat_type: kind,
         language: if matches!(
             chat_type,
-            broadcast_chat::CREATURE_TEXT_EMOTE | broadcast_chat::EMOTE
+            broadcast_chat::CREATURE_TEXT_EMOTE
+                | broadcast_chat::CREATURE_BOSS_EMOTE
+                | broadcast_chat::CREATURE_WHISPER
+                | broadcast_chat::EMOTE
         ) {
             Language::Universal
         } else {
@@ -481,6 +499,43 @@ mod tests {
         assert_eq!(weird.language, Language::Universal);
     }
 
+    #[test]
+    fn creature_speech_keeps_its_wire_kinds_and_languages() {
+        let make = |kind| {
+            build_chat_message_to(
+                11,
+                Some("Herald".into()),
+                77,
+                kind,
+                7,
+                "Attention".into(),
+                0,
+            )
+        };
+        let whisper = make(broadcast_chat::CREATURE_WHISPER);
+        assert!(
+            matches!(whisper.chat_type, SMSG_MESSAGECHAT_ChatType::MonsterWhisper { monster, .. } if monster.guid() == 11)
+        );
+        assert_eq!(whisper.language, Language::Universal);
+        let boss = make(broadcast_chat::CREATURE_BOSS_EMOTE);
+        assert!(matches!(
+            boss.chat_type,
+            SMSG_MESSAGECHAT_ChatType::RaidBossEmote { .. }
+        ));
+        assert_eq!(boss.language, Language::Universal);
+        let yell = make(broadcast_chat::CREATURE_ZONE_YELL);
+        assert!(
+            matches!(yell.chat_type, SMSG_MESSAGECHAT_ChatType::MonsterYell { target, .. } if target.guid() == 77)
+        );
+        assert_eq!(yell.language, Language::Common);
+        let emote = make(broadcast_chat::CREATURE_ZONE_EMOTE);
+        assert!(matches!(
+            emote.chat_type,
+            SMSG_MESSAGECHAT_ChatType::MonsterEmote { .. }
+        ));
+        assert_eq!(emote.language, Language::Common);
+    }
+
     /// A player `/e` row (no creature name — EventAI never emits EMOTE) builds `Emote { sender2 }`
     /// naming the speaker, in Universal whatever language byte the row carries
     /// (cm:Chat.cpp:3660-3662, cm:Player.cpp:16594).