diff --git a/.github/workflows/module-durable.yml b/.github/workflows/module-durable.yml index af1d98e90..5d61f49b8 100644 --- a/.github/workflows/module-durable.yml +++ b/.github/workflows/module-durable.yml @@ -1,4 +1,4 @@ -name: Module durable tests +name: Module durable tests and Gateway bindings # The durable integration rung: `module/tests/`, marked `#[ignore]` because each target # starts its own SpacetimeDB standalone, publishes the module to it, and drives real reducers. @@ -54,6 +54,14 @@ jobs: echo "spacetime $found" [ "$found" = "$SPACETIME_VERSION" ] + - name: Test the Gateway binding check + run: python3 -m unittest scripts/tests/test_check_gateway_bindings.py + + # This regenerates the complete private schema into a temporary directory. It does not need a + # running Realm and never publishes the Module. + - name: Check all Gateway bindings + run: scripts/check-gateway-bindings.py + # Building the workspace Module once keeps the module-path publishers off a cold compile, and # a build failure reports as a build failure rather than as a publish timeout. The typed taxi # test builds and privately copies the exact Wasm artifact it inspects. diff --git a/docs/danger-zones.md b/docs/danger-zones.md index 35e145161..808118dc9 100644 --- a/docs/danger-zones.md +++ b/docs/danger-zones.md @@ -36,6 +36,9 @@ any line in §1 needs a human review before it ships, whoever or whatever wrote code accesses; `cargo build` E0609 "no field X" tells you which way each one goes. Regeneration also removes the explanatory `facing` / `facing_angle` comment from `creature_spline_type.rs`; restore it from the committed binding after every regeneration. + `scripts/check-gateway-bindings.py` runs the same private generation offline with SpacetimeDB + 2.7.1, applies only these spelling and comment exceptions in its temporary output, formats it + with the pinned Rust toolchain, and compares every generated file with the committed tree. Regeneration needs a machine that can run `spacetimedb-standalone` — schema extraction shells out to it, so a sandbox without it cannot regenerate. Diff-review the regenerated tree before committing (it rewrites every file; drift deletions are expected and correct), then run the @@ -44,22 +47,11 @@ any line in §1 needs a human review before it ships, whoever or whatever wrote `gateway/tests/schema_parity.rs` structurally checks every subscribed table's binding against the module schema — run `cargo test -p lyracore-gateway` after ANY hand edit to a `_type.rs` file to catch a missed, misordered, or mistyped column before it ships. - **One documented exception:** `game_transfer_out` plus the seven transfer/instance reducer - bindings were spliced BY HAND rather than regenerating all ~512 files — they are the only - module→gateway data flow the cross-database transfer adds, and the regen's blast radius was - worse than the splice. `transfer_out_type.rs` / `game_transfer_out_table.rs` say so in their - headers, and `schema_parity.rs` covers the row shape. A future regen overwrites both with - identical content; nothing needs undoing. - **A second exception:** `record_shard_load`/`record_region_load` (and their `game_shard_load` / - `game_region_load` tables) were also hand-spliced, because no live `spacetimedb-standalone` was - available to run the generator. Only the REDUCER bindings were added - (`record_shard_load_reducer.rs`, `record_region_load_reducer.rs`, mirroring - `install_guid_range_reducer.rs`'s shape) — the gateway only ever WRITES those two tables (an - operator reads them with `spacetime sql`), so no TABLE binding exists and none is needed. A - future regen adds the two table bindings (harmless, since nothing subscribes them) and - overwrites the reducer bindings with equivalent content; nothing needs undoing. This is also - the case that establishes the general rule: **a table binding is only needed if the gateway - subscribes to or reads that table.** + `game_transfer_out`, its transfer/instance reducers, `record_shard_load`, and + `record_region_load` began as narrow hand-splices when a full regeneration was unavailable or + carried too much unrelated churn. The full generated baseline now contains equivalent output + for them, and the complete-tree check covers their table and reducer definitions. The Gateway + still only needs to use a table binding when it subscribes to or reads that table. **A defaulted column's default can be a VALID VALUE, not a sentinel — plan the backfill (#456).** `cell` was END-appended to the four AOI-scoped tables as `#[default(0i64)]`, and 0 is the legitimate diff --git a/scripts/check-gateway-bindings.py b/scripts/check-gateway-bindings.py new file mode 100755 index 000000000..0ce95041e --- /dev/null +++ b/scripts/check-gateway-bindings.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Regenerate every Gateway binding and compare it with the committed tree.""" + +import argparse +import difflib +import re +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + + +SPACETIME_VERSION = "2.7.1" +GENERATED_RELATIVE_PATH = Path("gateway/src/stdb/bindings") +IDENTIFIER_FIXES = { + "aura_type.rs": { + "eff_p_0_kind": "eff_p0_kind", + "eff_p_0": "eff_p0", + "eff_p_1": "eff_p1", + }, + "gossip_menu_profile_option_type.rs": { + "cond_value_1": "cond_value1", + "cond_value_2": "cond_value2", + }, + "gossip_option_type.rs": { + "cond_value_1": "cond_value1", + "cond_value_2": "cond_value2", + }, +} +FACING_FIELDS = " pub facing: bool,\n pub facing_angle: f32," +FACING_FIELDS_WITH_COMMENT = """ // Hand-appended under the END-append rule in docs/danger-zones.md: a facing-only leg does not + // move (`sx/sy/sz == dx/dy/dz`, `dur_ms == 0`). The angle is ignored unless `facing` is true. + pub facing: bool, + pub facing_angle: f32,""" + + +@dataclass(frozen=True) +class Drift: + missing: tuple[Path, ...] + unexpected: tuple[Path, ...] + changed: tuple[Path, ...] + + def found(self): + return bool(self.missing or self.unexpected or self.changed) + + +def files_below(root): + return {path.relative_to(root) for path in root.rglob("*") if path.is_file()} + + +def find_drift(committed, generated): + committed_files = files_below(committed) + generated_files = files_below(generated) + common = committed_files & generated_files + return Drift( + missing=tuple(sorted(committed_files - generated_files)), + unexpected=tuple(sorted(generated_files - committed_files)), + changed=tuple( + path + for path in sorted(common) + if (committed / path).read_bytes() != (generated / path).read_bytes() + ), + ) + + +def replace_generator_identifier(path, generated, committed): + generated_token = re.compile(rf"\b{re.escape(generated)}\b") + text = path.read_text() + if generated_token.search(text): + path.write_text(generated_token.sub(committed, text)) + elif not re.search(rf"\b{re.escape(committed)}\b", text): + raise RuntimeError(f"{path.name}: expected {generated} or {committed}") + + +def apply_documented_exceptions(generated): + for relative, fixes in IDENTIFIER_FIXES.items(): + path = generated / relative + for generated_name, committed_name in fixes.items(): + replace_generator_identifier(path, generated_name, committed_name) + + spline = generated / "creature_spline_type.rs" + text = spline.read_text() + if FACING_FIELDS_WITH_COMMENT not in text: + if text.count(FACING_FIELDS) != 1: + raise RuntimeError( + "creature_spline_type.rs: expected one facing/facing_angle field pair" + ) + spline.write_text(text.replace(FACING_FIELDS, FACING_FIELDS_WITH_COMMENT)) + + +def format_generated(root, generated): + paths = sorted(str(path) for path in generated.rglob("*.rs")) + for start in range(0, len(paths), 100): + subprocess.run( + ["rustfmt", "--edition", "2021", *paths[start : start + 100]], + cwd=root, + check=True, + ) + + +def check_spacetime_version(): + result = subprocess.run( + ["spacetime", "--version"], + check=True, + capture_output=True, + text=True, + ) + match = re.search(r"spacetimedb tool version ([0-9.]+)", result.stdout) + found = match.group(1) if match else "unknown" + if found != SPACETIME_VERSION: + raise RuntimeError( + f"spacetime {SPACETIME_VERSION} is required; found {found}" + ) + print(f"Using spacetime {found}", flush=True) + + +def generate_bindings(root, output): + check_spacetime_version() + result = subprocess.run( + [ + "spacetime", + "generate", + "--lang", + "rust", + "--out-dir", + str(output), + "--module-path", + str(root / "module"), + "--include-private", + "--build-options=--features=debug_reducers", + "-y", + ], + cwd=root, + capture_output=True, + text=True, + ) + if result.returncode: + sys.stderr.write(result.stdout) + sys.stderr.write(result.stderr) + raise RuntimeError("spacetime generate failed") + + +def print_drift(drift, committed, generated): + for path in drift.missing: + print(f"Missing generated binding: {path}", file=sys.stderr) + for path in drift.unexpected: + print(f"Uncommitted generated binding: {path}", file=sys.stderr) + for path in drift.changed: + print(f"Changed generated binding: {path}", file=sys.stderr) + before = (committed / path).read_text().splitlines(keepends=True) + after = (generated / path).read_text().splitlines(keepends=True) + diff = difflib.unified_diff( + before, + after, + fromfile=f"committed/{path}", + tofile=f"generated/{path}", + ) + for line in list(diff)[:200]: + print(line, end="", file=sys.stderr) + + +def check(root, committed, supplied_generated=None): + with tempfile.TemporaryDirectory(prefix="lyracore-gateway-bindings-") as directory: + generated = Path(directory) / "bindings" + if supplied_generated: + shutil.copytree(supplied_generated, generated) + else: + generate_bindings(root, generated) + apply_documented_exceptions(generated) + drift = find_drift(committed, generated) + if drift.missing or drift.unexpected: + print_drift(drift, committed, generated) + return 1 + format_generated(root, generated) + drift = find_drift(committed, generated) + if drift.found(): + print_drift(drift, committed, generated) + return 1 + print(f"All {len(files_below(committed))} Gateway binding files are current") + return 0 + + +def main(): + root = Path(__file__).resolve().parent.parent + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--generated-dir", + type=Path, + help="Check an existing generated tree instead of invoking spacetime", + ) + parser.add_argument( + "--bindings-dir", + type=Path, + default=root / GENERATED_RELATIVE_PATH, + help="Committed binding tree", + ) + args = parser.parse_args() + return check(root, args.bindings_dir, args.generated_dir) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_check_gateway_bindings.py b/scripts/tests/test_check_gateway_bindings.py new file mode 100644 index 000000000..6f96d5069 --- /dev/null +++ b/scripts/tests/test_check_gateway_bindings.py @@ -0,0 +1,58 @@ +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "check-gateway-bindings.py" +ROOT = SCRIPT.parent.parent +COMMITTED = ROOT / "gateway/src/stdb/bindings" +FIXTURE_FILE = Path("advance_taxi_flight_reducer.rs") + + +class BindingCheckTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.generated = Path(self.temporary.name) / "generated" + shutil.copytree(COMMITTED, self.generated) + + def tearDown(self): + self.temporary.cleanup() + + def run_check(self): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--generated-dir", + str(self.generated), + "--bindings-dir", + str(COMMITTED), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + + def test_omitted_generated_definition_fails_the_check(self): + (self.generated / FIXTURE_FILE).unlink() + + result = self.run_check() + + self.assertNotEqual(0, result.returncode) + self.assertIn(f"Missing generated binding: {FIXTURE_FILE}", result.stderr) + + def test_modified_generated_definition_fails_the_check(self): + with (self.generated / FIXTURE_FILE).open("a") as fixture: + fixture.write("\n// Drift fixture.\n") + + result = self.run_check() + + self.assertNotEqual(0, result.returncode) + self.assertIn(f"Changed generated binding: {FIXTURE_FILE}", result.stderr) + + +if __name__ == "__main__": + unittest.main()