From d60182018911d3f1fc236d355185592b2b6ec8ad Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 4 Oct 2026 23:19:28 +0800 Subject: [PATCH 01/11] docs: add a changelog generated by git-cliff CHANGELOG.md lists every release from 0.1.0 and ships in the sdist. The commit that bumps the version regenerates it, and the release workflow checks that it has a section for the tagged version. Two-digit group sort keys keep the Security group after Reverts. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 4 + CHANGELOG.md | 149 ++++++++++++++++++++++++++++++++++ README.en.md | 11 +++ README.md | 10 +++ cliff.toml | 22 ++--- pyproject.toml | 1 + 6 files changed, 186 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c870a5..ebb881a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,10 @@ jobs: echo "tag $GITHUB_REF_NAME, package $version and rust/Cargo.toml $cargo differ" >&2 exit 1 fi + if ! grep -qF "## [$version]" CHANGELOG.md; then + echo "CHANGELOG.md has no section for $version" >&2 + exit 1 + fi # one abi3 wheel (CPython 3.11 and later) per platform with the extension module nnnotes._deck, and the sdist wheels: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3c19c4b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,149 @@ +# Changelog + +All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/). + +## Unreleased + +### Features + +- **deck-data:** Export character rank experience thresholds ([3abe101](https://github.com/MetaSekaiLab/nnnotes/commit/3abe1017e47d267f77822af949500789c1831ee4)) +- **music-data:** Publish the recommendation engine in the replay manifest ([42098d7](https://github.com/MetaSekaiLab/nnnotes/commit/42098d7bfa538912bbc350beaebfe78f3b04f202)) + +### Bug fixes + +- **deck-data:** Export represented VIP rank identities ([c5d0514](https://github.com/MetaSekaiLab/nnnotes/commit/c5d051454f90c2afc7fb0063485f7dcf5d5a7669)) + +### Documentation + +- Add a changelog generated by git-cliff ([7e871c5](https://github.com/MetaSekaiLab/nnnotes/commit/7e871c5f4dda7bde9323e0a8a71fb11d3cabeba3)) + +### Build + +- **deck:** Build the deck model from the ournotes-sim crate ([946fb33](https://github.com/MetaSekaiLab/nnnotes/commit/946fb335aef234837c461dfa960f3ce28dc946da)) + +## [0.1.3](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.2...v0.1.3) - 2026-09-30 + +### Features + +- **songs:** Export every song's metadata and chart facts ([3dae671](https://github.com/MetaSekaiLab/nnnotes/commit/3dae671302ca843c2ce573802e1309b4b85893a2)) +- [**breaking**] One music-data command with the deck model built in ([197b436](https://github.com/MetaSekaiLab/nnnotes/commit/197b436dfccbefa5296fd40fcf5167458ab1406e)) +- **music-data:** Play scenarios: Gekisou off, every rank, the Perfect play ([a03591e](https://github.com/MetaSekaiLab/nnnotes/commit/a03591e7fd4f186df7ddbe161c787c7cad138b41)) +- **music-data:** Read decoded master data with its manifest (--decoded-master) ([12df2a6](https://github.com/MetaSekaiLab/nnnotes/commit/12df2a67b71ade5e41b51e6602d97d6b5733bd99)) +- **jp:** Support Japanese release assets and split APKs ([9bd4452](https://github.com/MetaSekaiLab/nnnotes/commit/9bd4452b262b349cf4ae7b20b4110f532f2ba8d4)) +- **music-data:** The Gekisou catalog, range luck points and the seed set check ([ffaa1bc](https://github.com/MetaSekaiLab/nnnotes/commit/ffaa1bc10eecc2269ffcffb5fa952e166dc1053f)) +- **music-data:** Export and validate single-skill Gekisou aptitude ([35ae7a0](https://github.com/MetaSekaiLab/nnnotes/commit/35ae7a0727934f9f12a8a0b1b650b3a42c6f9a5f)) +- Export versioned Rust replay data with chart statistics ([46b5b09](https://github.com/MetaSekaiLab/nnnotes/commit/46b5b09c7356c27874ee01c610dc295f60b48765)) +- **ui:** Export offline serialized prefab libraries ([0a7344d](https://github.com/MetaSekaiLab/nnnotes/commit/0a7344d09dbeddde36237b8bf63e83bee2159885)) + +### Bug fixes + +- **jp:** Address review regressions ([5f40d3e](https://github.com/MetaSekaiLab/nnnotes/commit/5f40d3ee2b195b802a97bf4a861e3f64d223d1f6)) +- **catalog:** Select international catalogs by resource version ([15cb3bf](https://github.com/MetaSekaiLab/nnnotes/commit/15cb3bfe079282f2624c5a0eeba757566c7eb5f6)) +- **release:** Preflight wheels and preserve reviewed notes ([b1e12c4](https://github.com/MetaSekaiLab/nnnotes/commit/b1e12c4ab31cfed6cd453e562aeeb66c3821af83)) + +### Build + +- **deck:** Pin the final Gekisou aptitude implementation ([4881a21](https://github.com/MetaSekaiLab/nnnotes/commit/4881a2183a1d1cef9ecc799f0bf293dbc58469d7)) + +### CI + +- Move the GitHub Actions to their current majors ([f0038c1](https://github.com/MetaSekaiLab/nnnotes/commit/f0038c15887b2ad8b61ddde320560837dbab1b5f)) +- Run commitlint on Node.js 24, the active LTS ([48b6557](https://github.com/MetaSekaiLab/nnnotes/commit/48b6557490f638f2b37b73dabb8fa175f0cce788)) + +## [0.1.2](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.1...v0.1.2) - 2026-09-28 + +### Bug fixes + +- **unity:** Treat a MonoBehaviour with a missing script as classless ([83dd161](https://github.com/MetaSekaiLab/nnnotes/commit/83dd161e7a2af6953e97c56548341ab6839def0c)) +- **room:** Write a valid glTF for a room without geometry ([8465dd5](https://github.com/MetaSekaiLab/nnnotes/commit/8465dd5d89d3a1857d4dcb80b9f5ff8a36895473)) +- **story:** Share a models directory between concurrent story runs ([2051001](https://github.com/MetaSekaiLab/nnnotes/commit/2051001924ecdbb39d6bc6c5d011a9b52d43a188)) + +### Documentation + +- **readme:** Completion after verifying every story, spot and region ([c4d6740](https://github.com/MetaSekaiLab/nnnotes/commit/c4d67403b434a05f011a247a5c5bf48f52ebb9d1)) + +## [0.1.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.0...v0.1.1) - 2026-09-27 + +### Features + +- **config:** Write, edit and check the config file ([1f9b1a2](https://github.com/MetaSekaiLab/nnnotes/commit/1f9b1a2ab9ecb1a29a328a88c8dbf351e010597e)) +- **web:** [**breaking**] Store site assets compressed, gzip by default ([e90a46b](https://github.com/MetaSekaiLab/nnnotes/commit/e90a46b19f332e137b0fa49316651e2fb014a464)) +- **story:** [**breaking**] Stories reference their Live2D models instead of embedding them ([510ff98](https://github.com/MetaSekaiLab/nnnotes/commit/510ff98d90d19bee91e6f87f555c5fbb4104e08b)) + +### Bug fixes + +- **room:** Share equal images and key objects by their file ([a8af00a](https://github.com/MetaSekaiLab/nnnotes/commit/a8af00a5f89afa11f0cdb0caf032afe38e6e1b18)) + +## [0.1.0](https://github.com/MetaSekaiLab/nnnotes/releases/tag/v0.1.0) - 2026-09-27 + +### Features + +- Add the nnnotes data toolkit ([073c60c](https://github.com/MetaSekaiLab/nnnotes/commit/073c60cc029b067e2b5db9a8c5772f7dbdcc58d4)) +- Read the APK boot data with bundled type trees ([fb1739e](https://github.com/MetaSekaiLab/nnnotes/commit/fb1739ebe5b898dcf5c6a4c02b4eb88f6f59ec28)) +- Build Live2D models into the web site ([03e3831](https://github.com/MetaSekaiLab/nnnotes/commit/03e38314eebf7d6a9bb62b5905778056b73d33c3)) +- Fetch the current master data version from the game's API ([bcd7f20](https://github.com/MetaSekaiLab/nnnotes/commit/bcd7f20d9bb1e3bfeb89dd72f49f5d360b8da2ae)) +- Serve every region and language from one web site ([8160121](https://github.com/MetaSekaiLab/nnnotes/commit/81601210452e3b40f195e115beaae0dccf10f7cc)) +- Add open and game text styles for exported text (--fonts) ([da41edf](https://github.com/MetaSekaiLab/nnnotes/commit/da41edf7fe17fff125f56d9a5d5ab2afa5e3f8f6)) +- Export frames, effects, stills, chat and videos of story episodes ([2bacea1](https://github.com/MetaSekaiLab/nnnotes/commit/2bacea1e02054393b4d8bc09d7325c3fa2be06da)) +- Add the asset export: export, plan, run-stage, catalogs ([d04423c](https://github.com/MetaSekaiLab/nnnotes/commit/d04423c2db72eb8d350c8271aaf2c9ea4bf64b88)) +- **live:** Offer the Live options that select files ([dc4f1e7](https://github.com/MetaSekaiLab/nnnotes/commit/dc4f1e704ccdebba878f179f24aa2157d6186c88)) +- **web:** Export story episodes for the story player ([197af04](https://github.com/MetaSekaiLab/nnnotes/commit/197af04954c4f5bd4dc6932140f57c0b7f937452)) +- **export:** FLAC level 8 by default and explicit layout placement ([11bf5d8](https://github.com/MetaSekaiLab/nnnotes/commit/11bf5d87f6decd8b0d465661c8d41f48b227945e)) +- **story:** Chat texts, emoji sprites, frame particles, lit rooms and CRI Lips data ([5eb4f6c](https://github.com/MetaSekaiLab/nnnotes/commit/5eb4f6cfa1978eab6846ff6cc99b48a997b90efc)) +- **deck-data:** One JSON of charts and master tables for deck-building tools ([06e6943](https://github.com/MetaSekaiLab/nnnotes/commit/06e6943e4a2ad977d6684871906f7d90a282a227)) +- **voices:** Voice index view and query command; one ACB table reader ([6bb748a](https://github.com/MetaSekaiLab/nnnotes/commit/6bb748aaa7e24942f5cee6ebb704e7c016e9a8c7)) +- **cri.movie:** All USM streams (MPEG-1, H.264, alpha, subtitles, HCA) and bundle-held movies ([ff97da8](https://github.com/MetaSekaiLab/nnnotes/commit/ff97da8b2cb01721041522f69d8c42c978b5150c)) +- **voices:** Story episode voices with their lines, speakers and cues ([e65d84d](https://github.com/MetaSekaiLab/nnnotes/commit/e65d84d9733e039bf046c3bfd56a5cda06eba118)) +- **story:** Missing glyphs and frame texts in the story font data ([476dff0](https://github.com/MetaSekaiLab/nnnotes/commit/476dff0c02f47017f26260997fb2b8bce0d2078e)) +- **story:** Open font assets follow the game's TMP fallback chains ([b5cd68e](https://github.com/MetaSekaiLab/nnnotes/commit/b5cd68ee7bbd3dc2dc58c9fb8858a20b34e7179c)) + +### Bug fixes + +- Correct the ADV resource closure for chat stamps and IgnoreData rows ([9f03fb9](https://github.com/MetaSekaiLab/nnnotes/commit/9f03fb97784ecdfbeb0e576c8e257c0e6123b05c)) +- Stop with a usage error on unknown keys, ids and charts ([1fa6958](https://github.com/MetaSekaiLab/nnnotes/commit/1fa69589a68246ffa2cbd5a96bda8c1646be37b1)) +- Read the region, CDN and bundle key only when a download needs them ([da1d47c](https://github.com/MetaSekaiLab/nnnotes/commit/da1d47c918d7895e2f83390cf69f888817d0fcb9)) +- Handle more mesh, texture and reference forms in room and spot ([4ddc76c](https://github.com/MetaSekaiLab/nnnotes/commit/4ddc76ca667d5458a77928560e562060a331cf6b)) +- Generate MonoBehaviour typetrees in Unity's own node layout ([d0efb2c](https://github.com/MetaSekaiLab/nnnotes/commit/d0efb2cbbe17a2749fdb7f8f6b2370d25de95917)) +- Story UI language, model names, master JSON and external AWB ([29aada6](https://github.com/MetaSekaiLab/nnnotes/commit/29aada69e763dde8c2fd51fe287dfea118314766)) +- **export:** Read the APK's raw files from the APK ([3d9f79c](https://github.com/MetaSekaiLab/nnnotes/commit/3d9f79c135afb232a12ed3661584f780187a9b4b)) +- Size default worker counts by the CPUs the process may use ([a65eaf8](https://github.com/MetaSekaiLab/nnnotes/commit/a65eaf89d9726dcd3200166c3483cded94b30633)) +- **web:** Name --story when the fonts extra is missing ([8cb0f66](https://github.com/MetaSekaiLab/nnnotes/commit/8cb0f66f8e2ffcbc0171bbd0fc1bfbac9be09baa)) + +### Performance + +- Reuse decoded textures, shader dumps and cue sheets by content ([1f7c166](https://github.com/MetaSekaiLab/nnnotes/commit/1f7c166d008eb06b0ac6f193ad7958fb88dbf7b4)) +- Pipeline the web build and reuse scenes, BGM decodes and read sets ([25461d1](https://github.com/MetaSekaiLab/nnnotes/commit/25461d12c715369285027d008ebc53f49c73acb6)) +- Start numpy with one OpenBLAS thread unless the environment sets it ([f93d3d2](https://github.com/MetaSekaiLab/nnnotes/commit/f93d3d2d4e1f34e863d96cf17175ead1376d09cc)) +- Serve read-set plans from long-lived player processes ([440ceaa](https://github.com/MetaSekaiLab/nnnotes/commit/440ceaad5c63e83bba4b3a2dde2592467b61c111)) +- **export:** Bound the memory of the finalize and of plan -o ([cad57e6](https://github.com/MetaSekaiLab/nnnotes/commit/cad57e61c4483300f402418501bfb754558287bf)) +- **cli:** Start without UnityPy; --flac-level on audio, live and story ([c5057d5](https://github.com/MetaSekaiLab/nnnotes/commit/c5057d5855b3703308c9b5d7d762821a0498bfaa)) + +### Refactoring + +- Rename the site command to web ([43fe4cc](https://github.com/MetaSekaiLab/nnnotes/commit/43fe4cc3e016aa8a773182f8e8995b1ef27b1066)) + +### Documentation + +- Remove the contributing guide ([056b4f6](https://github.com/MetaSekaiLab/nnnotes/commit/056b4f64316a5949f99f1cbefd3f0f1d7692a97e)) +- Title the project nnnotes？！ ([91b5042](https://github.com/MetaSekaiLab/nnnotes/commit/91b50422fc389945367924bcfdf773954fdafd03)) +- State determinism with its condition; settings per command ([9b0c6f7](https://github.com/MetaSekaiLab/nnnotes/commit/9b0c6f79b651303fca979b3811ffe62d9b7c0c00)) +- Recommend Python 3.13 ([cfea0f5](https://github.com/MetaSekaiLab/nnnotes/commit/cfea0f56be5758dc8b7fa3ca0f8f0191fc38336a)) +- Add performance.md: where a web build spends its time ([a4fb6a0](https://github.com/MetaSekaiLab/nnnotes/commit/a4fb6a0db5a6976630ac9cb069e026078772823a)) +- **performance:** Incremental runs and raw files not in the cache ([69a879c](https://github.com/MetaSekaiLab/nnnotes/commit/69a879ca1d64aecdd5eea3e352cd0861f0a88a66)) + +### Build + +- **release:** Publish to PyPI with trusted publishing ([bb542af](https://github.com/MetaSekaiLab/nnnotes/commit/bb542af0d7e22a0255a7ee45099a51eb7fbadd12)) + +### CI + +- Check the stage golden records with pinned library versions ([f2594f7](https://github.com/MetaSekaiLab/nnnotes/commit/f2594f70366c8b46d70d638168d43f6dd7949e5b)) + +### Miscellaneous + +- List MetaMiku and emptysekai as copyright holders ([1629616](https://github.com/MetaSekaiLab/nnnotes/commit/1629616da629b7fbe7f714287aeff17933176357)) +- List MetaMiku as an author ([cc43e13](https://github.com/MetaSekaiLab/nnnotes/commit/cc43e132875206d02d8665b19c06b27255ab0037)) +- Point the project links at MetaSekaiLab/nnnotes ([a2b8304](https://github.com/MetaSekaiLab/nnnotes/commit/a2b83040bcf06771028bbfd5e90c10c225bde3e9)) +- English package readme; check commits on pull requests only ([e602dd5](https://github.com/MetaSekaiLab/nnnotes/commit/e602dd5ebf6dda0f18da5ace54a02dbb893a796f)) + + diff --git a/README.en.md b/README.en.md index 0bfc5ed..0edad14 100644 --- a/README.en.md +++ b/README.en.md @@ -191,6 +191,17 @@ everywhere. See [docs/performance.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/performance.md) for where a `web` build spends its time and which parts of it run in compiled code. +## Changelog + +[CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) lists the changes in every version, +generated from the commits by [git-cliff](https://git-cliff.org/). The commit that bumps the version regenerates it: + +```bash +git cliff --tag vX.Y.Z -o CHANGELOG.md +``` + +When a `vX.Y.Z` tag is pushed, the release workflow checks the versions and that CHANGELOG.md has a section for it. + ## License MIT, see [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE). diff --git a/README.md b/README.md index d98d793..56299c9 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,16 @@ nnnotes web out/site --live2d adv_live2d_rana_003_casual_spring_01 --player Date: Mon, 5 Oct 2026 00:17:42 +0800 Subject: [PATCH 02/11] ci(commitlint): lift the line length limit of bodies and footers Commit bodies and footers wrap freely; the header keeps the 100-character limit of the conventional config. Co-Authored-By: Claude Opus 5.5 --- commitlint.config.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/commitlint.config.mjs b/commitlint.config.mjs index b29b5ae..fc66ba1 100644 --- a/commitlint.config.mjs +++ b/commitlint.config.mjs @@ -1,3 +1,8 @@ export default { extends: ["@commitlint/config-conventional"], + rules: { + // bodies and footers wrap freely; the header keeps its length limit + "body-max-line-length": [0], + "footer-max-line-length": [0], + }, }; From e7fdacdd34b80fdf391614f08fd5d7480d33904e Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 5 Oct 2026 00:18:44 +0800 Subject: [PATCH 03/11] fix(music-data): name the mismatch when an engine package is rejected A replay or recommendation engine package whose build.json has another format now says so, with the format it has and the one expected, instead of reporting a commit difference. A commit difference names both commits. A missing deck model identity is reported before the package files are read. Co-Authored-By: Claude Opus 5.5 --- src/nnnotes/replaydata.py | 11 +++++++---- tests/test_replaydata.py | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py index 9615d1d..1f9d864 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/replaydata.py @@ -29,8 +29,13 @@ def _package(files: dict[str, bytes], source: Path, model: dict | None, name: st built = json.loads((source / "build.json").read_text(encoding="utf8")) except (OSError, ValueError): raise deckdata.DeckDataError(f"{name}: missing or malformed build.json") from None - if built.get("format") != build_format or not model or built.get("commit") != model.get("commit"): - raise deckdata.DeckDataError(f"{name}: build.json commit differs from the pinned deck model") + if not model or not model.get("commit"): + raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required") + if built.get("format") != build_format: + raise deckdata.DeckDataError(f"{name}: build.json format is {built.get('format')!r}, not {build_format}") + if built.get("commit") != model["commit"]: + raise deckdata.DeckDataError(f"{name}: build.json commit {built.get('commit')!r} differs from the pinned " + f"deck model {model['commit']}") js_path, wasm_path, build_path = f"{prefix}/{js}", f"{prefix}/{wasm}", f"{prefix}/build.json" for file, path in ((js, js_path), (wasm, wasm_path)): if not (source / file).is_file(): @@ -38,8 +43,6 @@ def _package(files: dict[str, bytes], source: Path, model: dict | None, name: st files[path] = (source / file).read_bytes() if not files[wasm_path].startswith(b"\x00asm\x01\x00\x00\x00"): raise deckdata.DeckDataError(f"{name}: not a WASM v1 module") - if not model.get("commit"): - raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required") if built.get("jsSha256") != hashlib.sha256(files[js_path]).hexdigest() or \ built.get("wasmSha256") != hashlib.sha256(files[wasm_path]).hexdigest(): raise deckdata.DeckDataError(f"{name}: JS/WASM SHA differs from build.json") diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index f1f2276..b80b4ac 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -47,7 +47,8 @@ def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): (engine / "build.json").write_text(json.dumps(built)) stale = tmp_path / "stale" stale.mkdir() - with pytest.raises(musicdata.MusicDataError, match="commit differs"): + stale_pin = f"build.json commit '0{{40}}' differs from the pinned deck model {FakeDeck.COMMIT}" + with pytest.raises(musicdata.MusicDataError, match=stale_pin): export(stale, deck=FakeDeck(), replay_dir=stale / "replay", replay_engine=engine) assert not (stale / "music.json").exists() built["commit"] = FakeDeck.COMMIT @@ -128,8 +129,8 @@ def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): @pytest.mark.parametrize("change,message", [ - ("commit", "recommend engine: build.json commit differs"), - ("format", "recommend engine: build.json commit differs"), + ("commit", "recommend engine: build.json commit '0{40}' differs from the pinned deck model"), + ("format", "recommend engine: build.json format is 'ournotes.replay-engine/1', not ournotes.recommend-engine/1"), ("wasm", "recommend engine: JS/WASM SHA differs"), ("missing", "recommend engine: missing ournotes_recommend.js"), ("no-replay-dir", "--recommend-engine needs --replay-dir"), From a9d0bbd034aee00c696979eba3b1f33754b8a237 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 5 Oct 2026 00:16:30 +0800 Subject: [PATCH 04/11] feat(catalogs): record the resource version of an imported catalog `catalogs import` now records a resource version without `--resource-version`: a JP catalog's source version, else the version in an international file name `catalog__.bin` (none for `main`), the name `fetch` and the cache use. The version also becomes the default label. An invalid `--resource-version`, or one that differs from a JP catalog's source version, is an error. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 8 +++++--- src/nnnotes/cli_assets.py | 34 ++++++++++++++++++++++++++++++---- tests/test_cli_assets.py | 24 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/assets.md b/docs/assets.md index 15ce34c..e71c860 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -25,7 +25,7 @@ nnnotes export -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bun nnnotes plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] nnnotes run-stage TASK.json|DIR [...] [--store DIR] [--fetch] [--force] -nnnotes catalogs list [--json] | import FILE [--label L] [--apk-catalog FILE] | fetch [--label L] +nnnotes catalogs list [--json] | import FILE [--label L] [--apk-catalog FILE] [--resource-version V] | fetch [--label L] | diff OLD NEW [--json] nnnotes store verify [--quick] ``` @@ -109,8 +109,10 @@ The catalog versions of a store: `import` a catalog file (with the APK's catalog them (labels, version ids or sha256 prefixes): keys added, removed and changed, bundles and raw files by stable name, the keys that reach a changed file, and, when a run over each version has stored its address table, the objects by stable address (an object keeps its stable address while its object id changes with its bundle's content). A version -is labelled by `--label`, else by the resource version (`fetch`, when the region has an API root), else by the -first 12 hex digits of its sha256. `export` imports the current catalog when it is not imported yet. +is labelled by `--label`, else by its resource version, else by the first 12 hex digits of its sha256. `fetch` +records the resource version it downloads (none for `main`); `import` takes `--resource-version`, else the JP +catalog's source version, else the version in a file name `catalog__.bin` as `fetch` and the +cache name them (none for `main`). `export` imports the current catalog when it is not imported yet. ### store verify diff --git a/src/nnnotes/cli_assets.py b/src/nnnotes/cli_assets.py index b027ce4..2803f09 100644 --- a/src/nnnotes/cli_assets.py +++ b/src/nnnotes/cli_assets.py @@ -8,7 +8,7 @@ nnnotes plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] nnnotes run-stage TASK.json|DIR [...] [--store DIR] [--fetch] [--force] - nnnotes catalogs list [--json] | import FILE [--label L] | fetch [--label L] | diff A B [--json] + nnnotes catalogs list [--json] | import FILE [--label L] [--resource-version V] | fetch [--label L] | diff A B [--json] nnnotes store verify [--quick] The pipeline is the registry STAGE_SOURCES in order: each entry names the module and attribute of a stage (a Stage @@ -47,6 +47,7 @@ import argparse import fnmatch import json +import re import sys import threading import zipfile @@ -60,7 +61,7 @@ from .apkset import ApkSet from . import contract -from .config import ConfigError, describe as describe_setting, usable_cpus, use +from .config import ConfigError, check_catalog_version, describe as describe_setting, usable_cpus, use from .contract import Cost, IncompatibleTask, Input, Task from .stages import Env, Pending, Stage, describe, execute @@ -1335,6 +1336,28 @@ def _apk_catalog(args, cfg) -> bytes | None: return apk_catalog(apk) +CATALOG_FILE = re.compile(r"catalog_([A-Za-z0-9][A-Za-z0-9.-]*)_[A-Za-z0-9-]+\.bin") + + +def _import_resource_version(given: str | None, file: Path, source: dict | None) -> str | None: + """The resource version of an imported catalog: `given` (--resource-version), else the JP source's version, else + the version in an international file name `catalog__.bin` (none for `main`). A JP source is + authoritative: a different `given` is an error.""" + if given is not None: + try: + check_catalog_version(given) + except ValueError as e: + raise ValueError(f"--resource-version {given!r}: {e}") from None + if source is not None: + if given is not None and given != source["version"]: + raise ValueError(f"--resource-version {given} differs from the catalog's source version {source['version']}") + return source["version"] + if given is not None: + return given + m = CATALOG_FILE.fullmatch(file.name) + return m[1] if m and m[1] != "main" and ".." not in m[1] else None + + def cmd_catalogs_import(args, cfg, common): try: remote = Path(args.file).read_bytes() @@ -1345,9 +1368,10 @@ def cmd_catalogs_import(args, cfg, common): if cfg.provider() == "jp": from .jp import read_source source = read_source(args.file, remote).to_dict() + resource = _import_resource_version(args.resource_version, Path(args.file), source) v = _db(args, cfg).add(remote, _apk_catalog(args, cfg), label=args.label, region=cfg.get("catalog", "region"), language=cfg.get("catalog", "language"), - resource_version=args.resource_version, + resource_version=resource, apk_version_name=_apk_version(cfg.path("paths", "apk")), source=source) except ValueError as e: args.usage(str(e)) @@ -1525,7 +1549,9 @@ def bind(fn): m.add_argument("file", help="catalog__.bin") m.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") m.add_argument("--apk-catalog", help="the APK's catalog.bin (default: read from [paths] apk)") - m.add_argument("--resource-version", help="the game's resource version of this catalog") + m.add_argument("--resource-version", + help="the game's resource version of this catalog (default: the JP source's version, else the " + "version in the file name catalog__.bin)") _store_arg(m) m.set_defaults(func=bind(cmd_catalogs_import), usage=m.error) m = csub.add_parser("fetch", help="fetch the region's current catalog and import it") diff --git a/tests/test_cli_assets.py b/tests/test_cli_assets.py index 6a5587d..c4c6943 100644 --- a/tests/test_cli_assets.py +++ b/tests/test_cli_assets.py @@ -498,6 +498,30 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): assert nn(capsys, "catalogs", "diff", "v1", "v9", *s)[0] == 2 +def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, capsys): + s = ["--store", tmp_path / "s"] + named = tmp_path / "catalog_1.0.0.201_en.bin" + named.write_bytes(catalog_bytes()) + main = tmp_path / "catalog_main_en.bin" + main.write_bytes(synth.CatalogWriter().build([(NAMES["shared"], synth.remote(NAMES["shared"]), [])])) + assert nn(capsys, "catalogs", "import", named, *s)[0] == 0 + assert nn(capsys, "catalogs", "import", main, *s)[0] == 0 + vs = json.loads(nn(capsys, "catalogs", "list", "--json", *s)[1])["versions"] + assert [(v["labels"], v["resourceVersion"]) for v in vs] == [ + (["1.0.0.201"], "1.0.0.201"), ([contract.sha256(main.read_bytes())[:12]], None)] + + +def test_catalog_import_resource_version_option(): + file = Path("catalog_1.0.0.201_en.bin") + assert cli_assets._import_resource_version("1.0.0.300", file, None) == "1.0.0.300" + assert cli_assets._import_resource_version(None, Path("renamed.bin"), None) is None + assert cli_assets._import_resource_version(None, file, {"version": "1.0.0.300"}) == "1.0.0.300" + with pytest.raises(ValueError, match="differs from the catalog's source version"): + cli_assets._import_resource_version("1.0.0.201", file, {"version": "1.0.0.300"}) + with pytest.raises(ValueError, match="--resource-version"): + cli_assets._import_resource_version("../x", file, None) + + def test_store_verify(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") From 5c845eb365fcc23c36bf724f336ed747ba198057 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 5 Oct 2026 00:19:14 +0800 Subject: [PATCH 05/11] ci: test on Python 3.14 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4889741..443adb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 From 1e1bb179f98dddecb72465949276cbc4fb4e8fd7 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 5 Oct 2026 00:19:14 +0800 Subject: [PATCH 06/11] ci(deck): move create-pull-request to v8, which runs on Node.js 24 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml index ade1964..615efc3 100644 --- a/.github/workflows/deck.yml +++ b/.github/workflows/deck.yml @@ -38,7 +38,7 @@ jobs: run: | sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml cargo update -p ournotes-sim - - uses: peter-evans/create-pull-request@v7 + - uses: peter-evans/create-pull-request@v8 if: steps.deck.outputs.move == 'true' with: branch: deck/update From 3a6bdfc95dd80171724ac313e2aeb5278859a843 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Mon, 5 Oct 2026 00:37:52 +0800 Subject: [PATCH 07/11] ci(deck): dispatch CI and the commit message check on the pin pull request A pull request opened with the workflow token starts no workflow by itself, so the required checks never ran on the pull request that moves the deck pin. The Deck model workflow now dispatches CI and the commit message check on its branch after creating or updating the pull request, and their checks land on it. Both workflows accept workflow_dispatch; a dispatched commit message check lints the branch's commits since main. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 ++ .github/workflows/commitlint.yml | 19 ++++++++++++------- .github/workflows/deck.yml | 12 +++++++++++- tests/test_repo.py | 12 ++++++++++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 443adb6..61ab385 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [main] pull_request: + # the Deck model workflow dispatches CI on the pull request it opens + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 8bdfb96..321840c 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -1,10 +1,12 @@ name: Commit messages -# Pull requests only: the PR title and the PR's commits follow Conventional Commits. Commits pushed to main directly -# are not checked. +# Pull requests: the PR title and the PR's commits follow Conventional Commits. Commits pushed to main directly are +# not checked. Dispatched on a branch (the Deck model workflow does so for the pull request it opens): the branch's +# commits since main. on: pull_request: types: [opened, edited, synchronize, reopened] + workflow_dispatch: permissions: contents: read @@ -22,11 +24,14 @@ jobs: - name: Install commitlint run: npm install --no-save --no-package-lock @commitlint/cli@19 @commitlint/config-conventional@19 - name: Pull request title + if: github.event_name == 'pull_request' env: TITLE: ${{ github.event.pull_request.title }} run: printf '%s\n' "$TITLE" | npx --no-install commitlint --verbose - - name: Pull request commits - run: >- - npx --no-install commitlint --verbose - --from ${{ github.event.pull_request.base.sha }} - --to ${{ github.event.pull_request.head.sha }} + - name: Commits + env: + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + from=${BASE:-$(git merge-base origin/main "$HEAD")} + npx --no-install commitlint --verbose --from "$from" --to "$HEAD" diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml index 615efc3..8b8a57c 100644 --- a/.github/workflows/deck.yml +++ b/.github/workflows/deck.yml @@ -2,13 +2,15 @@ name: Deck model # nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml # pins, so that one nnnotes version always carries one deck model. When ournotes-deck's main moves past that commit, -# this opens a pull request that moves the pin; CI then builds and tests it like any change. +# this opens a pull request that moves the pin and dispatches CI and the commit message check on its branch (a pull +# request opened with the workflow token starts no workflow by itself); their checks land on the pull request. on: schedule: - cron: "17 3 * * *" workflow_dispatch: permissions: + actions: write contents: write pull-requests: write @@ -39,6 +41,7 @@ jobs: sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml cargo update -p ournotes-sim - uses: peter-evans/create-pull-request@v8 + id: pr if: steps.deck.outputs.move == 'true' with: branch: deck/update @@ -50,3 +53,10 @@ jobs: Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. + - name: Check the pull request + if: steps.pr.outputs.pull-request-operation == 'created' || steps.pr.outputs.pull-request-operation == 'updated' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run ci.yml --repo "$GITHUB_REPOSITORY" --ref deck/update + gh workflow run commitlint.yml --repo "$GITHUB_REPOSITORY" --ref deck/update diff --git a/tests/test_repo.py b/tests/test_repo.py index c1d9b08..1a9e75b 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -37,13 +37,21 @@ def workflow_triggers(text: str) -> list[str]: return events -def test_commit_messages_are_checked_on_pull_requests_only(): +def test_commit_messages_are_checked_on_pull_requests_and_dispatched_branches(): text = (ROOT / ".github" / "workflows" / "commitlint.yml").read_text(encoding="utf-8") - assert workflow_triggers(text) == ["pull_request"] + assert workflow_triggers(text) == ["pull_request", "workflow_dispatch"] assert "event_name == 'push'" not in text and "github.event.before" not in text assert "github.event.pull_request.title" in text and "github.event.pull_request.base.sha" in text +def test_the_deck_pin_pull_request_gets_its_checks(): + workflows = ROOT / ".github" / "workflows" + deck = (workflows / "deck.yml").read_text(encoding="utf-8") + for name in ("ci.yml", "commitlint.yml"): + assert "workflow_dispatch" in workflow_triggers((workflows / name).read_text(encoding="utf-8")) + assert f"gh workflow run {name} --repo \"$GITHUB_REPOSITORY\" --ref deck/update" in deck + + @pytest.mark.parametrize("name", ["README.md", "README.en.md"]) def test_determinism_is_stated_with_its_condition(name): text = (ROOT / name).read_text(encoding="utf-8") From 9f0ba0e234cc4119682b3be4ebcd85344e618b02 Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:39 +0000 Subject: [PATCH 08/11] build(deck): ournotes-deck ddf212b601731f9ab74245119d4408d2746a370c --- rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0608f52..c294017 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -96,7 +96,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=e27d289d549aff74955977659e1bee5c72d6a4f5#e27d289d549aff74955977659e1bee5c72d6a4f5" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=ddf212b601731f9ab74245119d4408d2746a370c#ddf212b601731f9ab74245119d4408d2746a370c" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 8986489..2e45adc 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned by commit; # .github/workflows/deck.yml opens a pull request when ournotes-deck's main moves -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "e27d289d549aff74955977659e1bee5c72d6a4f5" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] From 4fbd0eb41f2b17a50becfedf86e3621ba3ad0e0b Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 6 Oct 2026 09:14:14 +0800 Subject: [PATCH 09/11] ci(music-data): take the engines from the ournotes-deck release of the pinned model The build downloads ournotes-replay-wasm-v.tar.gz and ournotes-recommend-wasm-v.tar.gz of the ournotes-deck release of the ournotes-sim version rust/Cargo.lock pins, checks both against the release's SHA256SUMS and passes them to nnnotes as its replay and recommendation engines. The replay gate and the UI publisher check every engine of the replay manifest against the build-info.json of its release package: module, model commit and the SHA-256 of the web JS and WASM files. The build needs neither the deck source nor a WASM toolchain. Co-Authored-By: Claude Opus 5.5 --- .github/MUSIC_DATA.md | 12 +- .github/scripts/music_data.py | 108 +++++++------- .github/scripts/songs_page_data.py | 9 +- .github/scripts/test_music_data.py | 139 ++++++++---------- .../scripts/test_music_data_compression.py | 12 +- .github/workflows/music-data-region.yml | 16 -- 6 files changed, 128 insertions(+), 168 deletions(-) diff --git a/.github/MUSIC_DATA.md b/.github/MUSIC_DATA.md index 288ca17..a4cfeff 100644 --- a/.github/MUSIC_DATA.md +++ b/.github/MUSIC_DATA.md @@ -93,10 +93,10 @@ extend the existing linear plain-skill UI domain or derive source rank threshold `provenance.client`; nnnotes also reads the bundles the APK carries, as in the story site's builds), the decoded master data of moenotes-masterdata-sync (every file SHA-256 checked against `index.json`, `MasterManifest.json` included); - - the pinned ournotes-deck source (`MUSIC_DATA_DECK_SOURCE`, the commit `rust/Cargo.lock` pins, clean before and - after the builds): its CLI, then `wasm/replay` and `wasm/recommend`, each with its own lockfile, for - `wasm32-unknown-unknown` and through `wasm-bindgen --target web` (`ournotes_replay`, `ournotes_recommend`), each - package with a `build.json` of its commit and JS/WASM SHA-256; + - the replay and recommendation engines: `ournotes-replay-wasm-v.tar.gz` and + `ournotes-recommend-wasm-v.tar.gz` of the ournotes-deck release `v`, the version of the + `ournotes-sim` package `rust/Cargo.lock` pins, each checked against the release's `SHA256SUMS`; nnnotes takes + their web packages after checking each `build-info.json` (module, commit, file SHA-256) against its deck model; - `nnnotes music-data --decoded-master --jackets jackets --replay-dir replay --replay-engine ... --recommend-engine ... -o music-data.json`: the master data as decoded (no master key), `provenance.master` the manifest's version and SHA-256 of the files as served; the charts, cue @@ -152,8 +152,8 @@ nothing being published, `plan` finds no `build.json` and every run builds. The `replay` gate checks the replay inventory: every resource's SHA-256 and size, the immutable directory, the chart IDs, the replay engine and, required for a new build, `manifest.recommendEngine`: its model commit is the replay -engine's and the music data's, and its `build.json` (`ournotes.recommend-engine/1`, clean) names the JS and WASM -SHA-256. The recommendation engine's files are uploaded and read back before the manifest like every other resource. +engine's and the music data's. Each engine's `build-info.json`, from its ournotes-deck release package, names that +module, the model commit and the SHA-256 of the JS and WASM files. The recommendation engine's files are uploaded and read back before the manifest like every other resource. The replay inventory also accepts `manifest.snapLabels` (`nnnotes.replay-labels/1`). Its SHA/size, region, master version and all 13 served-table hashes must match the checked music-data provenance. It is uploaded and diff --git a/.github/scripts/music_data.py b/.github/scripts/music_data.py index e784a17..246fc40 100755 --- a/.github/scripts/music_data.py +++ b/.github/scripts/music_data.py @@ -10,7 +10,8 @@ master OUT every file of the snapshot of $MASTERDATA_REGION into OUT (SHA-256 checked against index.json, MasterManifest.json included) and its index entry as OUT.snapshot.json build OUT `nnnotes music-data --decoded-master --jackets OUT/jackets -o OUT/music-data.json` ([paths] - master: the master step's OUT), its printed summary in OUT/music-data.summary.json + master: the master step's OUT) with the replay and recommendation engines of the pinned + ournotes-deck release, its printed summary in OUT/music-data.summary.json check OUT MASTER PAGE the quality gates (.github/MUSIC_DATA.md) on OUT/music-data.json, with the published file as the baseline and PAGE the chart data page's modules (examples/songs) for the smoke test; the report in OUT/check.json, and OUT/build.json (the build marker) when every gate passed; @@ -52,7 +53,7 @@ BUILD_FORMAT = "moenotes.music-data-build/1" # This script's own version of a build: bump it when what it builds or publishes changes, so that the next run builds # although the master data, the deck model and nnnotes are the same. -RECIPE = 6 +RECIPE = 7 FILE, FILE_BR, MARKER, JACKETS, ARCHIVE = "music-data.json", "music-data.json.br", "build.json", "jackets/", "archive/" MANIFEST = "MasterManifest.json" SOURCE_PATHS = ("src", "rust", "pyproject.toml") # nnnotes' code: the commit that last changed one of them @@ -87,11 +88,10 @@ REPLAY_LABEL_TABLES = ("MasterSupportSkill", "MasterSupportSkillEffect", "MasterGekisouSupportSkill", "MasterGekisouSupportSkillEffect", "MasterText", "MasterSkillConditionSet", "MasterSkillCondition", "MasterSkillCumulativeCondition", "MasterSkillTarget", "MasterCharacter", "MasterMemberCard", "MasterSupportCard", "MasterBand") -# the web WASM packages built from the pinned model source: its package directory, the crate's WASM file, the -# wasm-bindgen --out-name, the build.json format and the build directory under the build step's OUT -ENGINES = (("wasm/replay", "ournotes_replay_wasm", "ournotes_replay", "ournotes.replay-engine/1", "replay-engine-build"), - ("wasm/recommend", "ournotes_recommend_wasm", "ournotes_recommend", "ournotes.recommend-engine/1", - "recommend-engine-build")) +# the web WASM packages (replay, recommendation) of the ournotes-deck release of the pinned deck model: the release +# tagged v of the ournotes-sim package in rust/Cargo.lock +DECK_RELEASES = "https://github.com/empty-sekai/ournotes-deck/releases/download" +ENGINES = ("replay", "recommend") SHA256 = re.compile(r"[0-9a-f]{64}") COMMIT = re.compile(r"[0-9a-f]{40}") LISTED = 20 # failures and warnings listed per gate @@ -155,20 +155,27 @@ def exists(key: str) -> bool: # ---------------------------------------------------------------- the inputs of a build -def deck_commit(root: Path = Path(".")) -> str: - """The ournotes-deck commit nnnotes builds its deck model with: the ournotes-sim package in rust/Cargo.lock.""" +def deck_package(root: Path = Path(".")) -> tuple[str, str]: + """The ournotes-sim package nnnotes builds its deck model with, in rust/Cargo.lock: its version and its + ournotes-deck commit.""" lock = root / "rust" / "Cargo.lock" if not lock.is_file(): fail("no rust/Cargo.lock: this nnnotes has no music-data command with the deck model (sync the fork with " "upstream, .github/MUSIC_DATA.md)") for block in lock.read_text(encoding="utf-8").split("[[package]]"): if re.search(r'^name = "ournotes-sim"$', block, re.M): + version = re.search(r'^version = "([^"]+)"$', block, re.M) m = re.search(r'^source = "git\+[^"#]*#([0-9a-f]{40})"$', block, re.M) - if m: - return m.group(1) + if version and m: + return version.group(1), m.group(1) fail("rust/Cargo.lock has no ournotes-sim git commit") +def deck_commit(root: Path = Path(".")) -> str: + """The ournotes-deck commit nnnotes builds its deck model with.""" + return deck_package(root)[1] + + def nnnotes_commit(root: Path = Path(".")) -> str: """The commit that last changed nnnotes' code (SOURCE_PATHS); the checkout needs its history.""" def git(*args): @@ -337,7 +344,7 @@ def cmd_build(out: str) -> None: usage = subprocess.run(nnnotes + ["music-data", "--help"], capture_output=True, text=True).stdout if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine")): fail("the installed nnnotes lacks decoded-master/replay/recommend export (sync the fork with upstream)") - engine, recommend = build_replay_engine(o) + engine, recommend = release_engines(o) if env("MASTERDATA_REGION") == "jp": import jp_vpngate jp_vpngate.start() @@ -357,38 +364,27 @@ def cmd_build(out: str) -> None: f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}") -def build_replay_engine(out: Path) -> tuple[Path, ...]: - """Build the CLI and the web WASM packages (ENGINES: replay, recommendation) from the same pinned clean model - source as nnnotes._deck; returns their directories, each with its build.json.""" - source = Path(env("MUSIC_DATA_DECK_SOURCE")).resolve() - pinned = deck_commit() - def check_source(): - head = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() - changed = subprocess.check_output(["git", "-C", str(source), "status", "--porcelain", "--untracked-files=all", - "--", "Cargo.toml", "Cargo.lock", "crates", - *(package for package, *_ in ENGINES)], text=True).strip() - if head != pinned or changed: - fail("replay model source is dirty or differs from nnnotes' pinned deck commit") - check_source() - subprocess.run(["cargo", "build", "--manifest-path", str(source / "Cargo.toml"), - "--release", "--locked", "-j2", "--bin", "ournotes-deck"], check=True) - target = out.parent / "replay-target" - engines = [] - for package, crate, name, _, directory in ENGINES: - engine = out / directory - engine.mkdir(parents=True, exist_ok=True) - subprocess.run(["cargo", "build", "--manifest-path", str(source / package / "Cargo.toml"), "--target-dir", - str(target), "--target", "wasm32-unknown-unknown", "--release", "--locked", "-j2"], check=True) - subprocess.run(["wasm-bindgen", "--target", "web", "--out-dir", str(engine), "--out-name", name, - str(target / f"wasm32-unknown-unknown/release/{crate}.wasm")], check=True) - engines.append(engine) - check_source() - for engine, (_, _, name, build_format, _) in zip(engines, ENGINES): - built = {"format": build_format, "commit": pinned, "workingTreeDirty": False, - "jsSha256": sha256((engine / f"{name}.js").read_bytes()), - "wasmSha256": sha256((engine / f"{name}_bg.wasm").read_bytes())} - (engine / "build.json").write_text(json.dumps(built, indent=2) + "\n", encoding="utf8") - return tuple(engines) +def release_engines(out: Path) -> tuple[Path, ...]: + """Download the web WASM packages (ENGINES) of the ournotes-deck release of the pinned deck model into OUT, each + checked against the release's SHA256SUMS; returns their paths. nnnotes checks each package's build-info.json + against the deck model it is built with.""" + version, _ = deck_package() + base = f"{DECK_RELEASES}/v{version}" + sums = {} + for line in get(f"{base}/SHA256SUMS").decode("utf-8").splitlines(): # sha256sum's " " + digest, _, name = line.partition(" ") + sums[name] = digest + packages = [] + for module in ENGINES: + name = f"ournotes-{module}-wasm-v{version}.tar.gz" + if name not in sums: + fail(f"ournotes-deck v{version} publishes no {name}") + data = get(f"{base}/{name}", timeout=300) + if sha256(data) != sums[name]: + fail(f"{name}: SHA-256 differs from the SHA256SUMS of ournotes-deck v{version}") + (out / name).write_bytes(data) + packages.append(out / name) + return tuple(packages) def replay_resources(out: Path, doc: dict) -> list[Path]: @@ -450,20 +446,22 @@ def local(base: Path, url: str) -> Path: expected_ids = sorted(c["scoreId"] for s in doc["songs"] for c in s["charts"]) if sorted(c["scoreId"] for c in manifest["charts"]) != expected_ids or pointer.get("charts") != len(expected_ids): raise ValueError("replay chart IDs/count differ from music data") - built = json.loads(local(manifest_path.parent, engine["build"]["url"]).read_bytes()) - if (built.get("format") != "ournotes.replay-engine/1" or built.get("commit") != engine["model"]["commit"] - or built.get("workingTreeDirty") is True or built.get("jsSha256") != engine["js"]["sha256"] - or built.get("wasmSha256") != engine["wasm"]["sha256"]): - raise ValueError("replay engine build identity is inconsistent or dirty") - if recommend is not None: - built = json.loads(local(manifest_path.parent, recommend["build"]["url"]).read_bytes()) - if (built.get("format") != "ournotes.recommend-engine/1" or built.get("commit") != recommend["model"]["commit"] - or built.get("workingTreeDirty") is not False or built.get("jsSha256") != recommend["js"]["sha256"] - or built.get("wasmSha256") != recommend["wasm"]["sha256"]): - raise ValueError("recommend engine build identity is inconsistent or dirty") + for module, entry in (("replay", engine), ("recommend", recommend)): + if entry is not None: + check_engine_build(module, entry, json.loads(local(manifest_path.parent, entry["build"]["url"]).read_bytes())) return paths + [manifest_path] +def check_engine_build(module: str, entry: dict, built: dict) -> None: + """An engine of a replay manifest against `built`, the build-info.json of its ournotes-deck release package: the + module, the model commit and the SHA-256 of the web JS and WASM files.""" + files = built.get("files") if isinstance(built.get("files"), dict) else {} + stem = f"web/ournotes_{module}_wasm" + if (built.get("kind") != "wasm" or built.get("module") != module or built.get("commit") != entry["model"]["commit"] + or files.get(f"{stem}.js") != entry["js"]["sha256"] or files.get(f"{stem}_bg.wasm") != entry["wasm"]["sha256"]): + raise ValueError(f"{module} engine differs from the build-info.json of its release package") + + # ---------------------------------------------------------------- the gates @dataclass class Context: diff --git a/.github/scripts/songs_page_data.py b/.github/scripts/songs_page_data.py index 7a0dde4..d52ee45 100644 --- a/.github/scripts/songs_page_data.py +++ b/.github/scripts/songs_page_data.py @@ -8,7 +8,7 @@ from pathlib import Path, PurePosixPath from http_compression import decode_content -from music_data import GATES, REPLAY_LABEL_TABLES +from music_data import GATES, REPLAY_LABEL_TABLES, check_engine_build BASE = "https://storage.bdon.moe/moenotes/music-data/" @@ -100,11 +100,8 @@ def main(): or not isinstance((labels.get("tables", {}).get(name) or {}).get("rows"), list) for name in REPLAY_LABEL_TABLES)): raise ValueError("published deck/labels differ from the frozen source") - engine_build = json.loads((out / str(manifest_path.parent / relative(manifest["engine"]["build"]["url"]))).read_bytes()) - if (engine_build["commit"] != model or engine_build.get("workingTreeDirty") is True or - engine_build["jsSha256"] != manifest["engine"]["js"]["sha256"] or - engine_build["wasmSha256"] != manifest["engine"]["wasm"]["sha256"]): - raise ValueError("published engine build metadata is inconsistent") + check_engine_build("replay", manifest["engine"], json.loads( + (out / str(manifest_path.parent / relative(manifest["engine"]["build"]["url"]))).read_bytes())) marker_sha = hashlib.sha256(marker_raw).hexdigest() fetch(PurePosixPath("build.json"), marker_sha, args.out / "marker-after.json", len(marker_raw)) pin = args.out / "page-pin" diff --git a/.github/scripts/test_music_data.py b/.github/scripts/test_music_data.py index cfa70af..8dae72a 100644 --- a/.github/scripts/test_music_data.py +++ b/.github/scripts/test_music_data.py @@ -553,12 +553,13 @@ def test_report_markdown(tmp_path): assert "- scenarios failure: chart 10 (100001 easy): offSeeds missing, expected exactly one" in text_ -def test_deck_commit_from_cargo_lock(tmp_path): +def test_deck_package_from_cargo_lock(tmp_path): lock = tmp_path / "rust" / "Cargo.lock" lock.parent.mkdir() lock.write_text('[[package]]\nname = "pyo3"\nversion = "0.29.0"\n\n[[package]]\nname = "ournotes-sim"\n' 'version = "0.0.1"\nsource = "git+https://github.com/empty-sekai/ournotes-deck?rev=' + "a" * 40 + "#" + "b" * 40 + '"\n', encoding="utf-8") + assert music_data.deck_package(tmp_path) == ("0.0.1", "b" * 40) assert music_data.deck_commit(tmp_path) == "b" * 40 with pytest.raises(SystemExit, match="sync the fork"): music_data.deck_commit(tmp_path / "nowhere") @@ -780,21 +781,23 @@ def resource(url, raw): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(raw) return {"url": url, "sha256": music_data.sha256(raw), "bytes": len(raw)} - js = resource("engine/ournotes_replay.js", b"export class ReplaySession {}") - wasm = resource("engine/ournotes_replay_bg.wasm", b"\0asm\x01\0\0\0") - built = resource("engine/build.json", json.dumps({"format": "ournotes.replay-engine/1", "commit": commit, - "jsSha256": js["sha256"], "wasmSha256": wasm["sha256"]}).encode()) + def engine(prefix, module, js_raw, wasm_raw): + # the files nnnotes takes from an ournotes-deck release package: web JS, web WASM and build-info.json + stem = f"ournotes_{module}_wasm" + js = resource(f"{prefix}/{stem}.js", js_raw) + wasm = resource(f"{prefix}/{stem}_bg.wasm", wasm_raw) + built = resource(f"{prefix}/build-info.json", json.dumps({ + "version": "0.0.2", "tag": "v0.0.2", "commit": commit, "kind": "wasm", "module": module, + "files": {f"web/{stem}.js": js["sha256"], f"web/{stem}_bg.wasm": wasm["sha256"]}}).encode()) + return {"model": {"commit": commit}, "js": js, "wasm": wasm, "build": built} charts = [{"scoreId": c["scoreId"], **resource(f"charts/{c['scoreId']}.json", b"{}")} for song in doc["songs"] for c in song["charts"]] manifest = {"format": "nnnotes.replay-manifest/1", "deckData": resource("deck-data.json", b"{}"), - "charts": charts, "engine": {"model": {"commit": commit}, "requestFormat": "ournotes.replay/1", - "js": js, "wasm": wasm, "build": built}} + "charts": charts, "engine": {"requestFormat": "ournotes.replay/1", **engine( + "engine", "replay", b"export class ReplaySession {}", b"\0asm\x01\0\0\0")}} if recommend: - js = resource("recommend/ournotes_recommend.js", b"export class RecommendationSession {}") - wasm = resource("recommend/ournotes_recommend_bg.wasm", b"\0asm\x01\0\0\0recommend") - built = resource("recommend/build.json", json.dumps({"format": "ournotes.recommend-engine/1", "commit": commit, - "workingTreeDirty": False, "jsSha256": js["sha256"], "wasmSha256": wasm["sha256"]}).encode()) - manifest["recommendEngine"] = {"model": {"commit": commit}, "js": js, "wasm": wasm, "build": built} + manifest["recommendEngine"] = engine("recommend", "recommend", b"export class RecommendationSession {}", + b"\0asm\x01\0\0\0recommend") raw_manifest = json.dumps(manifest).encode() resource("manifest.json", raw_manifest) doc["replay"] = {"format": manifest["format"], "manifestUrl": "replay/staging/manifest.json", @@ -814,12 +817,13 @@ def test_replay_resources_upload_before_document_and_marker(tmp_path, monkeypatc runtime = ["music-data/" + p.relative_to(out).as_posix() for p in paths] assert keys[-4:] == [runtime[-1], "music-data/music-data.json", "music-data/music-data.json.br", "music-data/build.json"] assert set(runtime[:-1]).issubset(keys[:-4]) - assert {replay_key(doc, f"recommend/{name}") for name in ("ournotes_recommend.js", "ournotes_recommend_bg.wasm", - "build.json")}.issubset(keys[:-4]) + assert {replay_key(doc, f"recommend/{name}") for name in ("ournotes_recommend_wasm.js", + "ournotes_recommend_wasm_bg.wasm", + "build-info.json")}.issubset(keys[:-4]) types = {k: mime for k, _, mime in s3.log} - assert types[replay_key(doc, "engine/ournotes_replay_bg.wasm")] == "application/wasm" - assert types[replay_key(doc, "recommend/ournotes_recommend_bg.wasm")] == "application/wasm" - assert types[replay_key(doc, "recommend/ournotes_recommend.js")] == "text/javascript" + assert types[replay_key(doc, "engine/ournotes_replay_wasm_bg.wasm")] == "application/wasm" + assert types[replay_key(doc, "recommend/ournotes_recommend_wasm_bg.wasm")] == "application/wasm" + assert types[replay_key(doc, "recommend/ournotes_recommend_wasm.js")] == "text/javascript" caches = {k: c for k, c, _ in s3.log} for path in paths: key = "music-data/" + path.relative_to(out).as_posix() @@ -946,8 +950,8 @@ def test_replay_labels_are_checked_and_uploaded_before_manifest(tmp_path, monkey assert keys.index(replay_key(doc, "snap-labels.json")) < keys.index(replay_key(doc, "manifest.json")) -@pytest.mark.parametrize("changed", ["bytes", "model", "escape", "absolute", "ids", "abi", "dirty", "recommend-bytes", - "recommend-model", "recommend-dirty", "recommend-format"]) +@pytest.mark.parametrize("changed", ["bytes", "model", "escape", "absolute", "ids", "abi", "module", "recommend-bytes", + "recommend-model", "recommend-commit", "recommend-files"]) def test_replay_identity_failure_prevents_any_upload(tmp_path, monkeypatch, changed): s3 = FakeS3() out, _ = published_out(tmp_path, monkeypatch, s3) @@ -965,27 +969,24 @@ def test_replay_identity_failure_prevents_any_upload(tmp_path, monkeypatch, chan manifest["charts"][0]["scoreId"] = -1 elif changed == "abi": manifest["engine"]["requestFormat"] = "other" - elif changed == "dirty": - built = replay_path(out, doc, "engine/build.json") - data = json.loads(built.read_bytes()) - data["workingTreeDirty"] = True - raw = json.dumps(data).encode() - built.write_bytes(raw) - manifest["engine"]["build"].update(sha256=music_data.sha256(raw), bytes=len(raw)) elif changed == "recommend-bytes": - replay_path(out, doc, "recommend/ournotes_recommend_bg.wasm").write_bytes(b"\0asm\x01\0\0\0other") + replay_path(out, doc, "recommend/ournotes_recommend_wasm_bg.wasm").write_bytes(b"\0asm\x01\0\0\0other") elif changed == "recommend-model": manifest["recommendEngine"]["model"]["commit"] = "other" - elif changed in ("recommend-dirty", "recommend-format"): - built = replay_path(out, doc, "recommend/build.json") + elif changed in ("module", "recommend-commit", "recommend-files"): + # a build-info.json that names another module, commit or file than the manifest's engine + key, prefix = ("engine", "engine") if changed == "module" else ("recommendEngine", "recommend") + built = replay_path(out, doc, f"{prefix}/build-info.json") data = json.loads(built.read_bytes()) - if changed == "recommend-dirty": - data["workingTreeDirty"] = True + if changed == "module": + data["module"] = "recommend" + elif changed == "recommend-commit": + data["commit"] = "b" * 40 else: - data["format"] = "ournotes.replay-engine/1" + data["files"]["web/ournotes_recommend_wasm_bg.wasm"] = "0" * 64 raw = json.dumps(data).encode() built.write_bytes(raw) - manifest["recommendEngine"]["build"].update(sha256=music_data.sha256(raw), bytes=len(raw)) + manifest[key]["build"].update(sha256=music_data.sha256(raw), bytes=len(raw)) rewrite_replay_manifest(out, doc, manifest) with pytest.raises(SystemExit, match="replay resources changed after gates"): music_data.cmd_publish(str(out)) @@ -996,7 +997,7 @@ def test_replay_read_back_failure_stops_before_runtime_pointer(tmp_path, monkeyp s3 = FakeS3() out, _ = published_out(tmp_path, monkeypatch, s3) doc, _ = replay_out(out) - s3.corrupt = replay_key(doc, "engine/ournotes_replay_bg.wasm") + s3.corrupt = replay_key(doc, "engine/ournotes_replay_wasm_bg.wasm") monkeypatch.setattr(music_data, "get", lambda *a, **kw: (_ for _ in ()).throw(music_data.urllib.error.URLError("offline"))) monkeypatch.setattr(music_data.http_compression, "get_object", lambda *a, **kw: (_ for _ in ()).throw(music_data.urllib.error.URLError("offline"))) with pytest.raises(SystemExit, match="does not serve what was uploaded"): @@ -1017,7 +1018,7 @@ def test_parallel_payload_readbacks_finish_before_any_pointer(tmp_path, monkeypa payloads = {archive, *(p.relative_to(out).as_posix() for p in paths[:-1])} pointers = [paths[-1].relative_to(out).as_posix(), music_data.FILE, music_data.FILE_BR, music_data.MARKER] if corrupt: - key = archive if corrupt == "archive" else replay_key(doc, "engine/ournotes_replay_bg.wasm").removeprefix("music-data/") + key = archive if corrupt == "archive" else replay_key(doc, "engine/ournotes_replay_wasm_bg.wasm").removeprefix("music-data/") s3.corrupt = "music-data/" + key monkeypatch.setattr(music_data, "get", lambda *a, **kw: (_ for _ in ()).throw(music_data.urllib.error.URLError("offline"))) monkeypatch.setattr(music_data.http_compression, "get_object", lambda *a, **kw: (_ for _ in ()).throw(music_data.urllib.error.URLError("offline"))) @@ -1046,49 +1047,27 @@ def read_back(b, key, digest): assert [key for key, _, _ in s3.log][-len(pointers):] == ["music-data/" + key for key in pointers] -@pytest.mark.parametrize("change", ["wrong-head", "untracked-source", "changed-during-build", None]) -def test_engine_build_checks_pinned_source_before_and_after_compiling(tmp_path, monkeypatch, change): - source, out = tmp_path / "source", tmp_path / "out" - source.mkdir() - out.mkdir() - monkeypatch.setenv("MUSIC_DATA_DECK_SOURCE", str(source)) - pinned = "a" * 40 - monkeypatch.setattr(music_data, "deck_commit", lambda: pinned) - checks = [] - builds = [] - def output(cmd, **kw): - if "rev-parse" in cmd: - return ("b" * 40 if change == "wrong-head" else pinned) + "\n" - checks.append(cmd) - return "?? src/untracked.rs\n" if change == "untracked-source" or ( - change == "changed-during-build" and len(checks) == 2) else "" - def run(cmd, **kw): - builds.append(cmd) - if cmd[0] == "wasm-bindgen": - engine, name = Path(cmd[cmd.index("--out-dir") + 1]), cmd[cmd.index("--out-name") + 1] - (engine / f"{name}.js").write_bytes(b"js " + name.encode()) - (engine / f"{name}_bg.wasm").write_bytes(b"wasm " + name.encode()) - monkeypatch.setattr(music_data.subprocess, "check_output", output) - monkeypatch.setattr(music_data.subprocess, "run", run) +@pytest.mark.parametrize("change", [None, "sha", "missing"]) +def test_engines_are_the_release_packages_of_the_pinned_model(tmp_path, monkeypatch, change): + monkeypatch.setattr(music_data, "deck_package", lambda: ("0.0.2", "a" * 40)) + packages = {f"ournotes-{module}-wasm-v0.0.2.tar.gz": f"package {module}".encode() for module in music_data.ENGINES} + sums = {name: music_data.sha256(raw) for name, raw in packages.items()} + if change == "sha": + sums["ournotes-recommend-wasm-v0.0.2.tar.gz"] = "0" * 64 + elif change == "missing": + del sums["ournotes-recommend-wasm-v0.0.2.tar.gz"] + base = f"{music_data.DECK_RELEASES}/v0.0.2/" + def get(url, timeout=120): + assert url.startswith(base) + name = url.removeprefix(base) + if name == "SHA256SUMS": + return "".join(f"{digest} {file}\n" for file, digest in sums.items()).encode() + return packages[name] + monkeypatch.setattr(music_data, "get", get) if change: - with pytest.raises(SystemExit, match="source is dirty or differs"): - music_data.build_replay_engine(out) - assert not (out / "replay-engine-build/build.json").exists() - assert not (out / "recommend-engine-build/build.json").exists() - assert len(builds) == (5 if change == "changed-during-build" else 0) - else: - replay, recommend = music_data.build_replay_engine(out) - assert all("wasm/replay" in cmd and "wasm/recommend" in cmd for cmd in checks) - assert [cmd[cmd.index("--manifest-path") + 1] for cmd in builds if cmd[0] == "cargo"][1:] == [ - str(source / "wasm/replay/Cargo.toml"), str(source / "wasm/recommend/Cargo.toml")] - bindgen = [cmd for cmd in builds if cmd[0] == "wasm-bindgen"] - assert all(cmd[1:3] == ["--target", "web"] for cmd in bindgen) - assert [Path(cmd[-1]).name for cmd in bindgen] == ["ournotes_replay_wasm.wasm", "ournotes_recommend_wasm.wasm"] - for engine, name, build_format in ((replay, "ournotes_replay", "ournotes.replay-engine/1"), - (recommend, "ournotes_recommend", "ournotes.recommend-engine/1")): - identity = json.loads((engine / "build.json").read_bytes()) - assert identity["format"] == build_format - assert identity["commit"] == pinned and identity["workingTreeDirty"] is False - assert identity["jsSha256"] == music_data.sha256(b"js " + name.encode()) - assert identity["wasmSha256"] == music_data.sha256(b"wasm " + name.encode()) - assert len(checks) == 2 and len(builds) == 5 + with pytest.raises(SystemExit, match="SHA-256 differs" if change == "sha" else "publishes no"): + music_data.release_engines(tmp_path) + return + replay, recommend = music_data.release_engines(tmp_path) + assert (replay.name, recommend.name) == ("ournotes-replay-wasm-v0.0.2.tar.gz", "ournotes-recommend-wasm-v0.0.2.tar.gz") + assert replay.read_bytes() == packages[replay.name] and recommend.read_bytes() == packages[recommend.name] diff --git a/.github/scripts/test_music_data_compression.py b/.github/scripts/test_music_data_compression.py index d658945..b8d0bd4 100644 --- a/.github/scripts/test_music_data_compression.py +++ b/.github/scripts/test_music_data_compression.py @@ -181,9 +181,10 @@ def test_ui_reader_freezes_new_snapshot_from_marker_archive_and_rejects_changed_ def resource(name, raw): blobs[name] = raw return {"url": name, "sha256": md.sha256(raw), "bytes": len(raw)} - js = resource("engine/replay.js", b"export default ()=>{}"); wasm = resource("engine/replay.wasm", b"wasm") - engine_build = resource("engine/build.json", json.dumps({"commit": model, "workingTreeDirty": False, - "jsSha256": js["sha256"], "wasmSha256": wasm["sha256"]}).encode()) + js = resource("engine/ournotes_replay_wasm.js", b"export default ()=>{}") + wasm = resource("engine/ournotes_replay_wasm_bg.wasm", b"wasm") + engine_build = resource("engine/build-info.json", json.dumps({"commit": model, "kind": "wasm", "module": "replay", + "files": {"web/ournotes_replay_wasm.js": js["sha256"], "web/ournotes_replay_wasm_bg.wasm": wasm["sha256"]}}).encode()) deck = resource("deck-data.json", json.dumps({"provenance": provenance}).encode()) labels = {"format": "nnnotes.replay-labels/1", "region": "tw", "masterVersion": source["masterVersion"], "tables": {name: {**table, "rows": []} for name, table in tables.items()}} @@ -266,8 +267,9 @@ def test_reencoding_changes_all_json_transport_and_preserves_wasm_js_and_every_d report = reencode.reencode(tmp_path / "encoding") assert report["encodedBytes"] < report["decodedBytes"] assert {"snap-labels.json", "deck-data.json", "manifest.json", "music-data.json", "build.json"} <= {Path(item["key"]).name for item in report["objects"]} - assert any(item["key"].endswith("/recommend/build.json") for item in report["objects"]) - assert {Path(key).name for key in report["unchangedArtifacts"]} >= {"ournotes_recommend.js", "ournotes_recommend_bg.wasm"} + assert any(item["key"].endswith("/recommend/build-info.json") for item in report["objects"]) + assert {Path(key).name for key in report["unchangedArtifacts"]} >= {"ournotes_recommend_wasm.js", + "ournotes_recommend_wasm_bg.wasm"} for full, raw in before.items(): if full.endswith(".json"): assert s3.headers[full]["ContentEncoding"] == "gzip" diff --git a/.github/workflows/music-data-region.yml b/.github/workflows/music-data-region.yml index 6d1c56d..611d107 100644 --- a/.github/workflows/music-data-region.yml +++ b/.github/workflows/music-data-region.yml @@ -119,21 +119,6 @@ jobs: # builds the extension module nnnotes._deck: the deck model at the commit rust/Cargo.toml pins run: uv pip install --system -e ".[test]" boto3 - - name: Pinned model revision - id: model - run: | - revision=$(python -c 'import sys; sys.path.insert(0, ".github/scripts"); import music_data; print(music_data.deck_commit())') - echo "revision=$revision" >> "$GITHUB_OUTPUT" - - uses: actions/checkout@v7 - with: - repository: empty-sekai/ournotes-deck - ref: ${{ steps.model.outputs.revision }} - path: pinned-replay-model - - name: WASM toolchain - run: | - rustup target add wasm32-unknown-unknown - cargo install wasm-bindgen-cli --version 0.2.127 --locked - - name: Gate self-test env: MUSIC_DATA_PAGE: ${{ env.WORK }}/page/examples/songs @@ -144,7 +129,6 @@ jobs: - name: Build env: - MUSIC_DATA_DECK_SOURCE: ${{ github.workspace }}/pinned-replay-model NNNOTES_BUNDLE_KEY: ${{ secrets.NNNOTES_BUNDLE_KEY }} NNNOTES_BUNDLE_NONCE_SEED: ${{ secrets.NNNOTES_BUNDLE_NONCE_SEED }} # A new cache on every run, never actions/cache: nnnotes keeps a downloaded catalog file for good (a cached From de34ee28ec181cac16237aa2751ff38e3bccfd91 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 6 Oct 2026 09:06:28 +0800 Subject: [PATCH 10/11] feat(music-data): take the deck model's WASM release packages as engines --replay-engine and --recommend-engine now take the ournotes-deck WASM release packages of the pinned model, as the downloaded ournotes--wasm-vVERSION.tar.gz or its directory. The package's build-info.json must describe that module's WASM package, name the pinned commit and list the SHA-256 of the web binding's JS and WASM files; the binding and build-info.json are copied byte for byte, so the published engine is the released one. The deck pin workflow follows ournotes-deck's releases: it moves the pin to the latest release once its replay and recommendation WASM packages are published. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deck.yml | 37 +++++++---- docs/music-data.md | 9 ++- docs/replay.md | 14 ++-- rust/Cargo.toml | 4 +- src/nnnotes/cli.py | 10 +-- src/nnnotes/replaydata.py | 79 +++++++++++++--------- tests/test_replaydata.py | 132 ++++++++++++++++++++----------------- 7 files changed, 163 insertions(+), 122 deletions(-) diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml index 8b8a57c..c0a7a97 100644 --- a/.github/workflows/deck.yml +++ b/.github/workflows/deck.yml @@ -1,9 +1,11 @@ name: Deck model # nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml -# pins, so that one nnnotes version always carries one deck model. When ournotes-deck's main moves past that commit, -# this opens a pull request that moves the pin and dispatches CI and the commit message check on its branch (a pull -# request opened with the workflow token starts no workflow by itself); their checks land on the pull request. +# pins, so that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the +# latest release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine +# and --recommend-engine) and its commit is past the pinned one, this opens a pull request that moves the pin and +# dispatches CI and the commit message check on its branch (a pull request opened with the workflow token starts no +# workflow by itself); their checks land on the pull request. on: schedule: - cron: "17 3 * * *" @@ -19,21 +21,29 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Compare the pinned commit with ournotes-deck's main + - name: Compare the pinned commit with ournotes-deck's latest release id: deck env: GH_TOKEN: ${{ github.token }} run: | pinned=$(sed -nE 's/^ournotes-sim = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml) [ -n "$pinned" ] || { echo "rust/Cargo.toml pins no ournotes-sim commit" >&2; exit 1; } - head=$(git ls-remote https://github.com/empty-sekai/ournotes-deck refs/heads/main | cut -f1) - # ahead: main contains the pinned commit and more; behind or diverged: leave the pin to a person (with no - # common history the comparison itself fails) + release=$(gh api repos/empty-sekai/ournotes-deck/releases/latest) + tag=$(jq -r .tag_name <<< "$release") + head=$(gh api "repos/empty-sekai/ournotes-deck/commits/$tag" --jq .sha) + published=true + for module in replay recommend; do + jq -e --arg name "ournotes-$module-wasm-$tag.tar.gz" 'any(.assets[]; .name == $name)' <<< "$release" \ + > /dev/null || published=false + done + # ahead: the release contains the pinned commit and more; behind or diverged: leave the pin to a person + # (with no common history the comparison itself fails) status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status) - echo "pinned $pinned, main $head: $status" + echo "pinned $pinned, $tag $head: $status, WASM packages published: $published" echo "pinned=$pinned" >> "$GITHUB_OUTPUT" echo "head=$head" >> "$GITHUB_OUTPUT" - echo "move=$([ "$status" = ahead ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "move=$([ "$status" = ahead ] && [ "$published" = true ] && echo true || echo false)" >> "$GITHUB_OUTPUT" - name: Move the pin if: steps.deck.outputs.move == 'true' working-directory: rust @@ -45,11 +55,12 @@ jobs: if: steps.deck.outputs.move == 'true' with: branch: deck/update - commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" - title: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" + title: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" body: | - Moves the deck model to ournotes-deck's main, - https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} + Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }} (${{ steps.deck.outputs.head }}), + https://github.com/empty-sekai/ournotes-deck/releases/tag/${{ steps.deck.outputs.tag }} + Changes: https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. diff --git a/docs/music-data.md b/docs/music-data.md index ef4caf9..7f30142 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -432,12 +432,15 @@ The deck model reads the same content under the format name `nnnotes.deck-data/1 ## Building -The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned by commit in `rust/Cargo.toml` +The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned to the commit of an ournotes-deck +release in `rust/Cargo.toml` (and `rust/Cargo.lock`) and built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/) (PyO3, the stable ABI of Python 3.11 and later: one wheel per platform). The release workflow builds the wheels; `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` -opens one when ournotes-deck's `main` moves), and `provenance.deck.commit` names it in every file. +opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in +every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM +packages. ## Versions @@ -454,4 +457,4 @@ their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags. -`--replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG]` writes normalized runtime inputs and pinned WASM assets (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. +`--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs and the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. diff --git a/docs/replay.md b/docs/replay.md index 867afc7..7a4c747 100644 --- a/docs/replay.md +++ b/docs/replay.md @@ -1,6 +1,6 @@ # Shared Rust/WASM replay inputs -`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. The replay engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json` (`format:"ournotes.replay-engine/1"`); the optional recommendation engine package, the wasm-bindgen `--target web` build of the deck model's `wasm/recommend`, must contain `ournotes_recommend.js`, `ournotes_recommend_bg.wasm` and `build.json` (`format:"ournotes.recommend-engine/1"`). Each `build.json` records `commit`, `jsSha256` and `wasmSha256`; the commit and both file hashes must match the measured model and the package files. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. +`nnnotes music-data --replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. Each engine is an ournotes-deck WASM release package of the pinned model, given as the downloaded archive or its extracted directory: `ournotes-replay-wasm-vVERSION.tar.gz` for `--replay-engine` and, optionally, `ournotes-recommend-wasm-vVERSION.tar.gz` for `--recommend-engine`. Its `build-info.json` must describe that module's WASM package (`kind:"wasm"`, `module`), name the pinned model's `commit`, and list in `files` the SHA-256 of the web binding's JS and WASM files, which must match the package files. The web binding (`web/ournotes__wasm.js` and `web/ournotes__wasm_bg.wasm`) and `build-info.json` are copied byte for byte. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. The music-data pointer is: @@ -18,15 +18,15 @@ The music-data pointer is: "engine": { "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"], - "js":{"url":"engine/ournotes_replay.js","sha256":"","bytes":0}, - "wasm":{"url":"engine/ournotes_replay_bg.wasm","sha256":"","bytes":0}, - "build":{"url":"engine/build.json","sha256":"","bytes":0} + "js":{"url":"engine/ournotes_replay_wasm.js","sha256":"","bytes":0}, + "wasm":{"url":"engine/ournotes_replay_wasm_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"engine/build-info.json","sha256":"","bytes":0} }, "recommendEngine": { "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, - "js":{"url":"recommend/ournotes_recommend.js","sha256":"","bytes":0}, - "wasm":{"url":"recommend/ournotes_recommend_bg.wasm","sha256":"","bytes":0}, - "build":{"url":"recommend/build.json","sha256":"","bytes":0} + "js":{"url":"recommend/ournotes_recommend_wasm.js","sha256":"","bytes":0}, + "wasm":{"url":"recommend/ournotes_recommend_wasm_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"recommend/build-info.json","sha256":"","bytes":0} }, "unlistedScoreIds":[], "clock":"Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2e45adc..58bfd60 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } -# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned by commit; -# .github/workflows/deck.yml opens a pull request when ournotes-deck's main moves +# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; +# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 3bc659e..14cc82b 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -1002,10 +1002,12 @@ def target(m, what): help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--replay-dir", metavar="DIR", help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory") - c.add_argument("--replay-engine", metavar="DIR", - help="copy pinned wasm-bindgen JS/WASM + build.json into --replay-dir") - c.add_argument("--recommend-engine", metavar="DIR", - help="copy the pinned wasm-bindgen recommendation JS/WASM + build.json into --replay-dir") + c.add_argument("--replay-engine", metavar="PKG", + help="the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its " + "directory); its web JS/WASM and build-info.json are copied into --replay-dir") + c.add_argument("--recommend-engine", metavar="PKG", + help="the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz " + "or its directory); its web JS/WASM and build-info.json are copied into --replay-dir") c.add_argument("--jackets", metavar="DIR", help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") _out(c, "output file (.json, or .json.gz for gzip)") diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py index 1f9d864..d345b6b 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/replaydata.py @@ -3,6 +3,7 @@ import hashlib import json +import tarfile from pathlib import Path from . import deckdata @@ -10,44 +11,62 @@ FORMAT = "nnnotes.replay-manifest/1" CHART_FORMAT = "nnnotes.replay-chart/1" -JS = "ournotes_replay.js" -WASM = "ournotes_replay_bg.wasm" -RECOMMEND_JS = "ournotes_recommend.js" -RECOMMEND_WASM = "ournotes_recommend_bg.wasm" +BUILD_INFO = "build-info.json" def _json(value) -> bytes: return deckdata.encode(deckdata._value(value, "replay input")) -def _package(files: dict[str, bytes], source: Path, model: dict | None, name: str, build_format: str, prefix: str, - js: str, wasm: str) -> tuple[str, str, str]: - """Copy a wasm-bindgen package of the pinned deck model (`js`, `wasm`, build.json) into `files` under `prefix`; - build.json must name the model's commit and both file hashes. Returns the JS, WASM and build.json paths.""" - source = Path(source) +def _read_package(source: Path, paths: tuple[str, ...], name: str) -> dict[str, bytes]: + """Those of the files `paths` that a release package holds: a directory, or a .tar.gz archive holding the package + directory.""" + if source.is_dir(): + return {path: (source / path).read_bytes() for path in paths if (source / path).is_file()} try: - built = json.loads((source / "build.json").read_text(encoding="utf8")) - except (OSError, ValueError): - raise deckdata.DeckDataError(f"{name}: missing or malformed build.json") from None + with tarfile.open(source, "r:gz") as archive: + return {member.name.partition("/")[2]: archive.extractfile(member).read() + for member in archive.getmembers() if member.isfile() and member.name.partition("/")[2] in paths} + except (OSError, tarfile.TarError): + raise deckdata.DeckDataError( + f"{name}: {source} is neither a package directory nor a .tar.gz package") from None + + +def _package(files: dict[str, bytes], source: Path, model: dict | None, module: str, + prefix: str) -> tuple[str, str, str]: + """Copy the web binding of an ournotes-deck WASM release package of `module` (`replay` or `recommend`), the + `ournotes--wasm-v` .tar.gz or its directory, into `files` under `prefix`. Its build-info.json must + describe the WASM package of `module` built from the pinned model's commit, with the SHA-256 of both files. + Returns the JS, WASM and build-info.json paths.""" + name = f"{module} engine" + stem = f"web/ournotes_{module}_wasm" + js, wasm = f"{stem}.js", f"{stem}_bg.wasm" + package = _read_package(Path(source), (BUILD_INFO, js, wasm), name) if not model or not model.get("commit"): raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required") - if built.get("format") != build_format: - raise deckdata.DeckDataError(f"{name}: build.json format is {built.get('format')!r}, not {build_format}") - if built.get("commit") != model["commit"]: - raise deckdata.DeckDataError(f"{name}: build.json commit {built.get('commit')!r} differs from the pinned " + if BUILD_INFO not in package: + raise deckdata.DeckDataError(f"{name}: missing {BUILD_INFO} in {source}") + try: + info = json.loads(package[BUILD_INFO]) + except ValueError: + raise deckdata.DeckDataError(f"{name}: malformed {BUILD_INFO}") from None + if not isinstance(info, dict) or info.get("kind") != "wasm" or info.get("module") != module: + raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} does not describe the {module} WASM package") + if info.get("commit") != model["commit"]: + raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} commit {info.get('commit')!r} differs from the pinned " f"deck model {model['commit']}") - js_path, wasm_path, build_path = f"{prefix}/{js}", f"{prefix}/{wasm}", f"{prefix}/build.json" - for file, path in ((js, js_path), (wasm, wasm_path)): - if not (source / file).is_file(): - raise deckdata.DeckDataError(f"{name}: missing {file} in {source}") - files[path] = (source / file).read_bytes() - if not files[wasm_path].startswith(b"\x00asm\x01\x00\x00\x00"): + hashes = info.get("files") if isinstance(info.get("files"), dict) else {} + for path in (js, wasm): + if path not in package: + raise deckdata.DeckDataError(f"{name}: missing {path} in {source}") + if hashes.get(path) != hashlib.sha256(package[path]).hexdigest(): + raise deckdata.DeckDataError(f"{name}: {path} SHA-256 differs from {BUILD_INFO}") + if not package[wasm].startswith(b"\x00asm\x01\x00\x00\x00"): raise deckdata.DeckDataError(f"{name}: not a WASM v1 module") - if built.get("jsSha256") != hashlib.sha256(files[js_path]).hexdigest() or \ - built.get("wasmSha256") != hashlib.sha256(files[wasm_path]).hexdigest(): - raise deckdata.DeckDataError(f"{name}: JS/WASM SHA differs from build.json") - files[build_path] = _json(built) - return js_path, wasm_path, build_path + paths = tuple(f"{prefix}/{path.rsplit('/', 1)[-1]}" for path in (js, wasm, BUILD_INFO)) + for path, data in zip(paths, (package[js], package[wasm], package[BUILD_INFO])): + files[path] = data + return paths def bundle(music: dict, engine_dir: Path | None = None, @@ -93,15 +112,13 @@ def resource(path: str) -> dict: model = music["provenance"].get("deck") engine = None if engine_dir is not None: - js, wasm, build = _package(files, engine_dir, model, "replay engine", "ournotes.replay-engine/1", "engine", - JS, WASM) + js, wasm, build = _package(files, engine_dir, model, "replay", "engine") engine = {"model": model, "requestFormat": "ournotes.replay/1", "class": "ReplaySession", "methods": ["describeChart", "template", "run"], "js": resource(js), "wasm": resource(wasm), "build": resource(build)} recommend = None if recommend_dir is not None: - js, wasm, build = _package(files, recommend_dir, model, "recommend engine", "ournotes.recommend-engine/1", - "recommend", RECOMMEND_JS, RECOMMEND_WASM) + js, wasm, build = _package(files, recommend_dir, model, "recommend", "recommend") recommend = {"model": model, "js": resource(js), "wasm": resource(wasm), "build": resource(build)} manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")}, "charts": charts, "engine": engine, diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index b80b4ac..dd1b02d 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -1,6 +1,7 @@ """Runtime inputs retain normalized note order/identities and bind the engine to the pinned model.""" import hashlib import json +import tarfile import pytest @@ -36,23 +37,43 @@ def test_replay_requires_actual_audio_length_and_contains_no_guess(tmp_path): assert not (tmp_path / "music.json").exists() +WASM_MAGIC = b"\x00asm\x01\x00\x00\x00" + + +def release_package(directory, module, commit=FakeDeck.COMMIT, js=None): + """A synthetic ournotes-deck WASM release package of `module`: its web binding and build-info.json.""" + js = js if js is not None else f"export class {module.title()}Session {{}}".encode() + web = directory / "web" + web.mkdir(parents=True, exist_ok=True) + stem = f"ournotes_{module}_wasm" + (web / f"{stem}.js").write_bytes(js) + (web / f"{stem}_bg.wasm").write_bytes(WASM_MAGIC) + info = {"version": "0.0.1", "tag": "v0.0.1", "commit": commit, "kind": "wasm", "module": module, + "target": "wasm32-unknown-unknown", + "files": {f"web/{stem}.js": hashlib.sha256(js).hexdigest(), + f"web/{stem}_bg.wasm": hashlib.sha256(WASM_MAGIC).hexdigest()}} + (directory / replaydata.BUILD_INFO).write_text(json.dumps(info, indent=2) + "\n") + return info + + +def archive(directory): + """The release archive of a package directory: the directory itself at the archive root.""" + path = directory.parent / f"{directory.name}.tar.gz" + with tarfile.open(path, "w:gz") as output: + output.add(directory, arcname=directory.name) + return path + + def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): - engine = tmp_path / "engine" - engine.mkdir() - js, wasm = b"export class ReplaySession {}", b"\x00asm\x01\x00\x00\x00" - (engine / replaydata.JS).write_bytes(js) - (engine / replaydata.WASM).write_bytes(wasm) - built = {"format": "ournotes.replay-engine/1", "commit": "0" * 40, - "jsSha256": hashlib.sha256(js).hexdigest(), "wasmSha256": hashlib.sha256(wasm).hexdigest()} - (engine / "build.json").write_text(json.dumps(built)) + engine = tmp_path / "ournotes-replay-wasm-v0.0.1" + release_package(engine, "replay", commit="0" * 40) stale = tmp_path / "stale" stale.mkdir() - stale_pin = f"build.json commit '0{{40}}' differs from the pinned deck model {FakeDeck.COMMIT}" + stale_pin = f"build-info.json commit '0{{40}}' differs from the pinned deck model {FakeDeck.COMMIT}" with pytest.raises(musicdata.MusicDataError, match=stale_pin): export(stale, deck=FakeDeck(), replay_dir=stale / "replay", replay_engine=engine) assert not (stale / "music.json").exists() - built["commit"] = FakeDeck.COMMIT - (engine / "build.json").write_text(json.dumps(built)) + release_package(engine, "replay") valid = tmp_path / "valid" valid.mkdir() result = export(valid, deck=FakeDeck(), replay_dir=valid / "replay", replay_engine=engine) @@ -60,54 +81,34 @@ def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): old_bytes = old_manifest_path.read_bytes() manifest = json.loads(old_bytes) assert manifest["engine"]["model"]["commit"] == FakeDeck.COMMIT - new_js = js + b"\n// updated engine" - (engine / replaydata.JS).write_bytes(new_js) - built["jsSha256"] = hashlib.sha256(new_js).hexdigest() - (engine / "build.json").write_text(json.dumps(built)) + release_package(engine, "replay", js=b"export class ReplaySession {}\n// updated engine") result = musicdata.export(valid / "music.json", deckdata.master_files(valid / "m"), KEY, CHARTS.__getitem__, bgm, **PROV, deck=musicdata.Deck(module=FakeDeck()), replay_dir=valid / "replay", replay_engine=engine) assert valid / result["replay"]["manifestUrl"] != old_manifest_path assert old_manifest_path.read_bytes() == old_bytes # readers of the preceding document keep its resource tree - (engine / replaydata.WASM).write_bytes(wasm + b"changed") + (engine / "web" / "ournotes_replay_wasm_bg.wasm").write_bytes(WASM_MAGIC + b"changed") changed = tmp_path / "changed" changed.mkdir() - with pytest.raises(musicdata.MusicDataError, match="SHA differs"): + with pytest.raises(musicdata.MusicDataError, match="ournotes_replay_wasm_bg.wasm SHA-256 differs"): export(changed, deck=FakeDeck(), replay_dir=changed / "replay", replay_engine=engine) -def package(directory, js_name, wasm_name, build_format, commit, js=b"export class Session {}"): - """A synthetic wasm-bindgen package with its build.json.""" - directory.mkdir(parents=True, exist_ok=True) - wasm = b"\x00asm\x01\x00\x00\x00" - (directory / js_name).write_bytes(js) - (directory / wasm_name).write_bytes(wasm) - built = {"format": build_format, "commit": commit, "workingTreeDirty": False, - "jsSha256": hashlib.sha256(js).hexdigest(), "wasmSha256": hashlib.sha256(wasm).hexdigest()} - (directory / "build.json").write_text(json.dumps(built)) - return built - - -def replay_package(tmp_path): - return package(tmp_path / "replay-pkg", replaydata.JS, replaydata.WASM, "ournotes.replay-engine/1", FakeDeck.COMMIT) - - -def recommend_package(tmp_path, commit=FakeDeck.COMMIT, build_format="ournotes.recommend-engine/1"): - return package(tmp_path / "recommend-pkg", replaydata.RECOMMEND_JS, replaydata.RECOMMEND_WASM, build_format, commit, - b"export class RecommendationSession {}") - - -def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): - replay_package(tmp_path) - built = recommend_package(tmp_path) +@pytest.mark.parametrize("packed", [False, True]) +def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path, packed): + replay = tmp_path / "ournotes-replay-wasm-v0.0.1" + recommend = tmp_path / "ournotes-recommend-wasm-v0.0.1" + release_package(replay, "replay") + info = release_package(recommend, "recommend") + replay_source, recommend_source = (archive(replay), archive(recommend)) if packed else (replay, recommend) plain = tmp_path / "plain" plain.mkdir() - result = export(plain, deck=FakeDeck(), replay_dir=plain / "replay", replay_engine=tmp_path / "replay-pkg") + result = export(plain, deck=FakeDeck(), replay_dir=plain / "replay", replay_engine=replay_source) assert "recommendEngine" not in json.loads((plain / result["replay"]["manifestUrl"]).read_bytes()) out = tmp_path / "out" out.mkdir() - result = export(out, deck=FakeDeck(), replay_dir=out / "replay", replay_engine=tmp_path / "replay-pkg", - recommend_engine=tmp_path / "recommend-pkg") + result = export(out, deck=FakeDeck(), replay_dir=out / "replay", replay_engine=replay_source, + recommend_engine=recommend_source) manifest_path = out / result["replay"]["manifestUrl"] manifest = json.loads(manifest_path.read_bytes()) entry = manifest["recommendEngine"] @@ -116,38 +117,45 @@ def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): assert entry["model"] == manifest["engine"]["model"] == data["provenance"]["deck"] assert entry["model"]["commit"] == FakeDeck.COMMIT assert [entry[k]["url"] for k in ("js", "wasm", "build")] == [ - "recommend/ournotes_recommend.js", "recommend/ournotes_recommend_bg.wasm", "recommend/build.json"] - assert manifest["engine"]["build"]["url"] == "engine/build.json" - for key, name in (("js", replaydata.RECOMMEND_JS), ("wasm", replaydata.RECOMMEND_WASM)): + "recommend/ournotes_recommend_wasm.js", "recommend/ournotes_recommend_wasm_bg.wasm", "recommend/build-info.json"] + assert [manifest["engine"][k]["url"] for k in ("js", "wasm", "build")] == [ + "engine/ournotes_replay_wasm.js", "engine/ournotes_replay_wasm_bg.wasm", "engine/build-info.json"] + for key, name in (("js", "ournotes_recommend_wasm.js"), ("wasm", "ournotes_recommend_wasm_bg.wasm")): payload = (manifest_path.parent / entry[key]["url"]).read_bytes() - assert payload == (tmp_path / "recommend-pkg" / name).read_bytes() + assert payload == (recommend / "web" / name).read_bytes() assert (entry[key]["sha256"], entry[key]["bytes"]) == (hashlib.sha256(payload).hexdigest(), len(payload)) - assert (entry["js"]["sha256"], entry["wasm"]["sha256"]) == (built["jsSha256"], built["wasmSha256"]) + assert entry[key]["sha256"] == info["files"][f"web/{name}"] build = (manifest_path.parent / entry["build"]["url"]).read_bytes() - assert json.loads(build) == built + assert build == (recommend / replaydata.BUILD_INFO).read_bytes() assert (entry["build"]["sha256"], entry["build"]["bytes"]) == (hashlib.sha256(build).hexdigest(), len(build)) @pytest.mark.parametrize("change,message", [ - ("commit", "recommend engine: build.json commit '0{40}' differs from the pinned deck model"), - ("format", "recommend engine: build.json format is 'ournotes.replay-engine/1', not ournotes.recommend-engine/1"), - ("wasm", "recommend engine: JS/WASM SHA differs"), - ("missing", "recommend engine: missing ournotes_recommend.js"), + ("commit", "recommend engine: build-info.json commit '0{40}' differs from the pinned deck model"), + ("module", "recommend engine: build-info.json does not describe the recommend WASM package"), + ("wasm", "recommend engine: web/ournotes_recommend_wasm_bg.wasm SHA-256 differs from build-info.json"), + ("missing", "recommend engine: missing web/ournotes_recommend_wasm.js"), + ("not-a-package", "recommend engine: .* is neither a package directory nor a .tar.gz package"), ("no-replay-dir", "--recommend-engine needs --replay-dir"), ]) def test_recommend_engine_rejects_another_build(tmp_path, change, message): - replay_package(tmp_path) - recommend_package(tmp_path, commit="0" * 40 if change == "commit" else FakeDeck.COMMIT, - build_format="ournotes.replay-engine/1" if change == "format" else "ournotes.recommend-engine/1") + replay = tmp_path / "replay-pkg" + release_package(replay, "replay") pkg = tmp_path / "recommend-pkg" + release_package(pkg, "recommend", commit="0" * 40 if change == "commit" else FakeDeck.COMMIT) + if change == "module": + pkg = tmp_path / "other-pkg" + release_package(pkg, "replay") if change == "wasm": - (pkg / replaydata.RECOMMEND_WASM).write_bytes(b"\x00asm\x01\x00\x00\x00changed") + (pkg / "web" / "ournotes_recommend_wasm_bg.wasm").write_bytes(WASM_MAGIC + b"changed") if change == "missing": - (pkg / replaydata.RECOMMEND_JS).unlink() - replay = {} if change == "no-replay-dir" else {"replay_dir": tmp_path / "replay", - "replay_engine": tmp_path / "replay-pkg"} + (pkg / "web" / "ournotes_recommend_wasm.js").unlink() + if change == "not-a-package": + pkg = tmp_path / "recommend.tar.gz" + pkg.write_bytes(b"not an archive") + options = {} if change == "no-replay-dir" else {"replay_dir": tmp_path / "replay", "replay_engine": replay} with pytest.raises(musicdata.MusicDataError, match=message): - export(tmp_path, deck=FakeDeck(), recommend_engine=pkg, **replay) + export(tmp_path, deck=FakeDeck(), recommend_engine=pkg, **options) assert not (tmp_path / "music.json").exists() and not (tmp_path / "replay").exists() From 8a088f4b7e40a79b3bc111e88c25d0abce397f36 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 6 Oct 2026 09:06:52 +0800 Subject: [PATCH 11/11] build(deck): ournotes-deck v0.0.2 Moves the deck model to ournotes-deck v0.0.2 (76e96b617842bdcc031cce98e3dfeead27ddbb85), https://github.com/empty-sekai/ournotes-deck/releases/tag/v0.0.2 Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. Co-Authored-By: Claude Opus 5.5 --- rust/Cargo.lock | 4 ++-- rust/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c294017..78763a6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" -version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=ddf212b601731f9ab74245119d4408d2746a370c#ddf212b601731f9ab74245119d4408d2746a370c" +version = "0.0.2" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=76e96b617842bdcc031cce98e3dfeead27ddbb85#76e96b617842bdcc031cce98e3dfeead27ddbb85" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 58bfd60..44aca0f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release]