ci: run durable Module and Coordinator tests

.github/workflows/module-durable.yml

@@ -0,0 +1,81 @@
+name: Module durable tests
+
+# The durable integration rung: `module/tests/`, marked `#[ignore]` because each target
+# starts its own SpacetimeDB standalone, publishes the module to it, and drives real reducers.
+# `rust.yml`'s `--lib` run cannot reach any of it.
+#
+# There is no shared node to start here: `module/tests/support/mod.rs` starts one standalone per
+# test on a free loopback port with `--in-memory`, publishes with `--features=debug_reducers` (the
+# feature set `lyracore publish` bakes in), and kills it on drop. This job installs the pinned CLI
+# and gets out of the way.
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+env:
+  # docs/quickstart.md and install.sh pin this exactly; `lyracore preflight` enforces the match.
+  SPACETIME_VERSION: 2.7.1
+  # Digest of the x86_64 Linux archive in the upstream v2.7.1 GitHub release.
+  SPACETIME_ARCHIVE_SHA256: 9b58192135733083479f30aba47dbbbc79b51db368c46cc221baa663e9c3a2c1
+
+jobs:
+  durable:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+
+      # Each test uses its own temporary CLI configuration.
+      - name: Install SpacetimeDB ${{ env.SPACETIME_VERSION }}
+        run: |
+          archive_path="$RUNNER_TEMP/spacetime.tar.gz"
+          cli_dir="$RUNNER_TEMP/spacetime-$SPACETIME_VERSION"
+          curl --fail --location --retry 3 \
+            "https://github.com/clockworklabs/SpacetimeDB/releases/download/v$SPACETIME_VERSION/spacetime-x86_64-unknown-linux-gnu.tar.gz" \
+            --output "$archive_path"
+          printf '%s  %s\n' "$SPACETIME_ARCHIVE_SHA256" "$archive_path" | sha256sum --check --strict
+          mkdir -p "$cli_dir"
+          tar --extract --gzip --file "$archive_path" --directory "$cli_dir"
+          ln -s spacetimedb-cli "$cli_dir/spacetime"
+          echo "$cli_dir" >> "$GITHUB_PATH"
+
+      # Verify the downloaded CLI matches the version required by the Module.
+      - name: Check the pinned version
+        run: |
+          found=$(spacetime --version | sed -n 's/.*version[ ]*\([0-9][0-9.]*\).*/\1/p' | head -n 1)
+          echo "spacetime $found"
+          [ "$found" = "$SPACETIME_VERSION" ]
+
+      # Building the workspace Module once keeps the module-path publishers off a cold compile, and
+      # a build failure reports as a build failure rather than as a publish timeout. The typed taxi
+      # test builds and privately copies the exact Wasm artifact it inspects.
+      - name: Build the module wasm
+        run: cargo build --release --target wasm32-unknown-unknown -p lyracore-module --features=debug_reducers
+
+      # Each test owns a whole database process, so the thread cap is a memory bound, not a CPU one.
+      #
+      # `deadmines` and `playerbots` are Packages an Operator installs (`lyracore packages add`);
+      # they are not in this repository, so their reducers do not exist in the published module.
+      # `lethal_floor`'s ranged phase races the module's own cast-event sweep and health regen, and
+      # is flaky on `main` with or without this change.
+      - name: Run the durable suite
+        run: cargo test -p lyracore-module --tests -- --ignored --test-threads=4 --skip deadmines --skip personality --skip lethal_floor
+
+      - name: Run the single-shard auction Refusal regression
+        run: cargo test -p lyracore-gateway --bin lyracore-gateway stdb::reducers::auction_reducer_tests::a_local_bid_interaction_refusal_stays_a_client_outcome -- --ignored --exact
+
+      - name: Upload the standalone logs
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: standalone-logs
+          path: /tmp/lyracore-standalone-logs
+          if-no-files-found: ignore

docs/danger-zones.md

@@ -369,6 +369,7 @@ cargo build
 #   importer 128 | module 603 | gateway 673 + 63 (schema parity) | lyracore-shared 75
 # These numbers drift UP as tests are added; a LOWER count is the signal worth chasing, not an exact
 # match.
+# The `#[ignore]` durable rung (`module/tests/`) now runs in CI: .github/workflows/module-durable.yml
 
 # Publish-shaped PREFLIGHT — the deploy-only break class the test suites cannot see.
 # Fully offline: no node, no database, no publish/call/sql. Three checks: the SpacetimeDB version

gateway/src/stdb/reducers.rs

@@ -3554,7 +3554,7 @@ mod auction_reducer_tests {
                 "unset {variable} for this private test"
             );
         }
-        let standalone = Standalone::start("auction-refusal");
+        let mut standalone = Standalone::start("auction-refusal");
         standalone.publish_module();
         standalone.assert_call("claim_operator", &[]);
         let cfg = GatewayConfig {

module/tests/auction_buyout.rs

@@ -6,14 +6,15 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn real_realm_reducer_commits_exact_buyout_mail_before_the_next_transaction() {
-    let standalone = Standalone::start("auction-buyout");
+    let mut standalone = Standalone::start("auction-buyout");
     standalone.publish_module();
     for (reducer, args) in [
         ("claim_operator", &[][..]),
         ("debug_stage_auction_buyout_fixture", &[][..]),
         (
             "realm_auction_decide_bid",
-            &["5090050", "5090051", "5090050", "900"][..],
+            // operation_id, bidder_guid, auction_id, house, offer — the fixture lists in house 1.
+            &["5090050", "5090051", "5090050", "1", "900"][..],
         ),
         ("debug_verify_auction_buyout_fixture", &[][..]),
     ] {

module/tests/auction_expiry.rs

@@ -6,13 +6,13 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn scheduled_bid_expiry_settles_once_and_a_callback_replay_is_a_no_op() {
-    let standalone = Standalone::start("auction-expiry");
+    let mut standalone = Standalone::start("auction-expiry");
     standalone.publish_module();
     for reducer in ["claim_operator", "debug_stage_auction_expiry_fixture"] {
         standalone.assert_call(reducer, &[]);
     }
 
-    standalone.wait_until_call_succeeds("debug_verify_auction_expiry_fixture");
+    standalone.wait_until_call_succeeds("debug_verify_auction_expiry_fixture", &[]);
 
     for reducer in [
         "debug_replay_auction_expiry_fixture",

module/tests/aura_tick.rs

@@ -11,7 +11,7 @@ const WOLF: u64 = (0xF130_u64 << 48) | (51_000_u64 << 24) | 1;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn a_periodic_kill_commits_the_death_and_an_unrelated_heal() {
-    let standalone = Standalone::start("aura-tick-death");
+    let mut standalone = Standalone::start("aura-tick-death");
     standalone.publish_module();
     standalone.assert_sql("DELETE FROM game_creature_move_schedule");
     standalone.assert_sql("DELETE FROM game_melee_schedule");

module/tests/deadmines_package.rs

@@ -10,7 +10,7 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI, the Wasm toolchain, and the dungeons Package installed (lyracore packages add dungeons)"]
 fn deadmines_choreography_reaches_package_owned_outcomes() {
-    let standalone = Standalone::start("deadmines-package");
+    let mut standalone = Standalone::start("deadmines-package");
     standalone.publish_module_anonymous();
     standalone.assert_call_anonymous("debug_deadmines_begin", &[]);
     standalone.assert_call_anonymous("debug_deadmines_rhahkzor_falls", &[]);

module/tests/eventai_quest_credit.rs

@@ -8,7 +8,7 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn eventai_quest_credit_keeps_pinned_recipient_and_completion_contracts() {
-    let standalone = Standalone::start("eventai-quest-credit");
+    let mut standalone = Standalone::start("eventai-quest-credit");
     standalone.publish_module();
     for (reducer, args) in [
         ("claim_operator", &[][..]),

module/tests/eventai_revision.rs

@@ -1,20 +1,22 @@
 mod support;
 
-use std::thread;
 use std::time::{Duration, Instant};
 
-use support::Standalone;
+use support::{poll_until, Standalone, POLL_TIMEOUT};
 
 const PLAYER_GUID: u64 = 1;
 const WOLF_ENTRY: u32 = 51_000;
 const WOLF_GUID: u64 = (0xF130_u64 << 48) | ((WOLF_ENTRY as u64) << 24) | 1;
 const SOURCE_RULE_ID: u64 = 900_001;
+/// `tick_creatures` fires on the seeded catch-all row every 0.5 s. The module counts no scheduled
+/// visits anywhere, so the only way to span one is to hold a window this wide open.
+const WORLD_TICK: Duration = Duration::from_millis(500);
 
 /// Runs only when requested because it builds and publishes the Wasm module to its own standalone.
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn scheduled_visits_preserve_state_then_clean_a_reloaded_definition() {
-    let standalone = Standalone::start("eventai-revision");
+    let mut standalone = Standalone::start("eventai-revision");
     standalone.publish_module();
     standalone.assert_call("debug_spawn_player_entity", &[&PLAYER_GUID.to_string()]);
 
@@ -26,8 +28,9 @@ fn scheduled_visits_preserve_state_then_clean_a_reloaded_definition() {
 
     wait_for_verifier(&standalone, true);
 
-    thread::sleep(Duration::from_millis(1_200));
-    assert_verifier(&standalone, true);
+    // Every scheduled visit inside this window is checked, not just the last one, so a visit that
+    // clobbers the adopted state fails here the moment it lands.
+    hold_verifier(&standalone, true, 2 * WORLD_TICK);
 
     let changed_rules = format!("{SOURCE_RULE_ID},aggro,100,4294967295,once,all,ordinary,phase:7");
     stage_fixture(&standalone, &changed_rules);
@@ -48,30 +51,37 @@ fn stage_fixture(standalone: &Standalone, rules: &str) {
 }
 
 fn wait_for_verifier(standalone: &Standalone, expect_rule_state: bool) {
-    let deadline = Instant::now() + Duration::from_secs(10);
-    loop {
+    let mut last = None;
+    let reached = poll_until(POLL_TIMEOUT, || {
         let output = verifier(standalone, expect_rule_state);
-        if output.status.success() {
+        let ok = output.status.success();
+        last = Some(output);
+        ok
+    });
+    let last = last.expect("at least one attempt");
+    if !reached {
+        standalone.assert_output_success(
+            &last,
+            "EventAI fixture did not reach the expected state before the deadline",
+        );
+    }
+}
+
+/// Re-check the verifier for `window`. Each check is a reducer call, which paces the loop. The last
+/// check lands at or after the end of the window, so a visit in its final milliseconds is seen.
+fn hold_verifier(standalone: &Standalone, expect_rule_state: bool, window: Duration) {
+    let until = Instant::now() + window;
+    loop {
+        assert_verifier(standalone, expect_rule_state);
+        if Instant::now() >= until {
             return;
         }
-        assert!(
-            Instant::now() < deadline,
-            "EventAI fixture did not reach the expected state\nstdout:\n{}\nstderr:\n{}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr),
-        );
-        thread::sleep(Duration::from_millis(50));
     }
 }
 
 fn assert_verifier(standalone: &Standalone, expect_rule_state: bool) {
     let output = verifier(standalone, expect_rule_state);
-    assert!(
-        output.status.success(),
-        "EventAI fixture verification failed\nstdout:\n{}\nstderr:\n{}",
-        String::from_utf8_lossy(&output.stdout),
-        String::from_utf8_lossy(&output.stderr),
-    );
+    standalone.assert_output_success(&output, "EventAI fixture verification failed");
 }
 
 fn verifier(standalone: &Standalone, expect_rule_state: bool) -> std::process::Output {

module/tests/eventai_runtime_boundaries.rs

@@ -6,7 +6,7 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn empty_relay_catalogue_and_spell_guardian_cleanup_apply_through_real_reducers() {
-    let standalone = Standalone::start("eventai-runtime-boundaries");
+    let mut standalone = Standalone::start("eventai-runtime-boundaries");
     standalone.publish_module_anonymous();
     standalone.assert_call_anonymous("debug_verify_eventai_spell_guardian_cleanup", &[]);
 }

module/tests/lethal_damage_floor.rs

@@ -1,8 +1,5 @@
 mod support;
 
-use std::thread;
-use std::time::Duration;
-
 use support::Standalone;
 
 const PLAYER_GUID: u64 = 1;
@@ -16,7 +13,7 @@ const SOURCE_RULE_ID: u64 = 900_001;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn authoritative_damage_honors_the_lethal_floor_and_its_lifetime() {
-    let standalone = Standalone::start("lethal-damage-floor");
+    let mut standalone = Standalone::start("lethal-damage-floor");
     standalone.publish_module();
     standalone.assert_call("debug_spawn_player_entity", &[&PLAYER_GUID.to_string()]);
     let first_definition = definition("phase:1");
@@ -117,8 +114,8 @@ fn authoritative_damage_honors_the_lethal_floor_and_its_lifetime() {
         "debug_set_lethal_damage_floor_health_fixture",
         &[&THIRD_WOLF_GUID.to_string(), "5"],
     );
-    thread::sleep(Duration::from_millis(1_300));
-    standalone.assert_call(
+    // The ranged swing lands on a scheduled visit, so the verifier itself is the condition.
+    standalone.wait_until_call_succeeds(
         "debug_verify_ranged_lethal_damage_floor_fixture",
         &[
             &PLAYER_GUID.to_string(),

module/tests/loot_tag.rs

@@ -6,7 +6,7 @@ use support::Standalone;
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn loot_tag_owns_threat_lifetime_rewards_and_corpse_eligibility() {
-    let standalone = Standalone::start("loot-tag");
+    let mut standalone = Standalone::start("loot-tag");
     standalone.publish_module_anonymous();
     standalone.assert_call_anonymous("debug_verify_loot_tag_fixture", &[]);
 }

module/tests/package_config.rs

@@ -37,7 +37,7 @@ fn config_value(rows: &[BTreeMap<String, String>], key: &str) -> Option<String>
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn debug_seed_package_config_seeds_once_and_never_clobbers_a_live_value_on_reseed() {
-    let standalone = Standalone::start("package-config-seed");
+    let mut standalone = Standalone::start("package-config-seed");
     standalone.publish_module();
     standalone.assert_call("claim_operator", &[]);
 
@@ -81,7 +81,7 @@ fn debug_seed_package_config_seeds_once_and_never_clobbers_a_live_value_on_resee
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn set_package_config_updates_an_existing_keys_value() {
-    let standalone = Standalone::start("package-config-update");
+    let mut standalone = Standalone::start("package-config-update");
     standalone.publish_module();
     standalone.assert_call("claim_operator", &[]);
     standalone.assert_call(
@@ -104,7 +104,7 @@ fn set_package_config_updates_an_existing_keys_value() {
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn set_package_config_refuses_an_unknown_key_and_names_the_packages_known_keys() {
-    let standalone = Standalone::start("package-config-refusal");
+    let mut standalone = Standalone::start("package-config-refusal");
     standalone.publish_module();
     standalone.assert_call("claim_operator", &[]);
     standalone.assert_call(
@@ -144,7 +144,7 @@ fn set_package_config_refuses_an_unknown_key_and_names_the_packages_known_keys()
 #[test]
 #[ignore = "requires the SpacetimeDB 2.7.1 CLI and Wasm toolchain"]
 fn set_package_config_with_allow_new_creates_an_unknown_key() {
-    let standalone = Standalone::start("package-config-allow-new");
+    let mut standalone = Standalone::start("package-config-allow-new");
     standalone.publish_module();
     standalone.assert_call("claim_operator", &[]);
     standalone.assert_call(