diff --git a/.github/MUSIC_DATA.md b/.github/MUSIC_DATA.md index a4cfeff..89c5825 100644 --- a/.github/MUSIC_DATA.md +++ b/.github/MUSIC_DATA.md @@ -98,7 +98,10 @@ extend the existing linear plain-skill UI domain or derive source rank threshold `ournotes-sim` package `rust/Cargo.lock` pins, each checked against the release's `SHA256SUMS`; nnnotes takes their web packages after checking each `build-info.json` (module, commit, file SHA-256) against its deck model; - `nnnotes music-data --decoded-master --jackets jackets --replay-dir replay --replay-engine ... --recommend-engine - ... -o music-data.json`: the master data as decoded (no + ... --stats-cache $MUSIC_DATA_STATS_CACHE -o music-data.json`: the deck statistics of the last build of the + region come back from `actions/cache` (derived numbers the file publishes, no game files), and only the charts + whose deck model sources (`provenance.deck.sourceSha256`), options, master tables or chart changed are measured + again; the summary counts both. The master data as decoded (no master key), `provenance.master` the manifest's version and SHA-256 of the files as served; the charts, cue sheets and jackets from the TW catalog, downloaded afresh on every run (never `actions/cache`: nnnotes keeps a downloaded catalog for good, and the cache holds decrypted game files); diff --git a/.github/scripts/music_data.py b/.github/scripts/music_data.py index 246fc40..5fe1561 100755 --- a/.github/scripts/music_data.py +++ b/.github/scripts/music_data.py @@ -11,7 +11,8 @@ index.json, MasterManifest.json included) and its index entry as OUT.snapshot.json build OUT `nnnotes music-data --decoded-master --jackets OUT/jackets -o OUT/music-data.json` ([paths] master: the master step's OUT) with the replay and recommendation engines of the pinned - ournotes-deck release, its printed summary in OUT/music-data.summary.json + ournotes-deck release and the chart statistics cache $MUSIC_DATA_STATS_CACHE, its + printed summary in OUT/music-data.summary.json check OUT MASTER PAGE the quality gates (.github/MUSIC_DATA.md) on OUT/music-data.json, with the published file as the baseline and PAGE the chart data page's modules (examples/songs) for the smoke test; the report in OUT/check.json, and OUT/build.json (the build marker) when every gate passed; @@ -342,15 +343,18 @@ def cmd_build(out: str) -> None: o.mkdir(parents=True, exist_ok=True) nnnotes = [sys.executable, "-m", "nnnotes"] usage = subprocess.run(nnnotes + ["music-data", "--help"], capture_output=True, text=True).stdout - if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine")): - fail("the installed nnnotes lacks decoded-master/replay/recommend export (sync the fork with upstream)") + if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine", + "--stats-cache")): + fail("the installed nnnotes lacks decoded-master/replay/recommend export or the statistics cache (sync the " + "fork with upstream)") engine, recommend = release_engines(o) if env("MASTERDATA_REGION") == "jp": import jp_vpngate jp_vpngate.start() cmd = nnnotes + ["music-data", "--decoded-master", "--jackets", str(o / "jackets"), "--replay-dir", str(o / "replay"), "--replay-engine", str(engine), - "--recommend-engine", str(recommend), "-o", str(o / FILE)] + "--recommend-engine", str(recommend), "--stats-cache", env("MUSIC_DATA_STATS_CACHE"), + "-o", str(o / FILE)] print("+ " + " ".join(cmd[1:]), flush=True) with open(o / "music-data.summary.json", "wb") as f: status = subprocess.run(cmd, stdout=f).returncode @@ -360,8 +364,10 @@ def cmd_build(out: str) -> None: import brotli (o / FILE_BR).write_bytes(brotli.compress(raw, quality=9)) r = json.loads((o / "music-data.summary.json").read_text(encoding="utf-8")) + stats = r.get("deckStats") or {} summary(f"- built: {r.get('songs')} songs, {r.get('charts')} charts, {r.get('jackets')} jackets, deck " - f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}") + f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}; chart statistics " + f"{stats.get('cached')} cached, {stats.get('measured')} measured") def release_engines(out: Path) -> tuple[Path, ...]: diff --git a/.github/workflows/music-data-region.yml b/.github/workflows/music-data-region.yml index 611d107..292d23d 100644 --- a/.github/workflows/music-data-region.yml +++ b/.github/workflows/music-data-region.yml @@ -38,6 +38,10 @@ env: PYTHON_VERSION: "3.13" PYTHONPATH: src WORK: ${{ github.workspace }}/music-data-work + # The deck statistics of every chart of the last build, under the SHA-256 of the model sources, options, master + # tables and chart they come from (nnnotes music-data --stats-cache): derived numbers the file publishes, no game + # files. A build measures only the charts whose inputs changed. + MUSIC_DATA_STATS_CACHE: ${{ github.workspace }}/music-data-work/stats-cache # Bundle keys come from secrets; catalog versions and CDN roots come from the saved master snapshot. NNNOTES_CATALOG_REGION: ${{ inputs.region == 'jp' && 'jp' || 'tw' }} NNNOTES_CATALOG_LANGUAGE: ${{ inputs.region == 'jp' && 'ja' || 'zh-Hant' }} @@ -127,6 +131,13 @@ jobs: - name: Master data run: python .github/scripts/music_data.py master "$WORK/master" + - name: Chart statistics of the last build + uses: actions/cache/restore@v6 + with: + path: ${{ env.MUSIC_DATA_STATS_CACHE }} + key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: music-data-stats-${{ inputs.region }}- + - name: Build env: NNNOTES_BUNDLE_KEY: ${{ secrets.NNNOTES_BUNDLE_KEY }} @@ -140,6 +151,12 @@ jobs: if [ "${{ inputs.region }}" = "jp" ]; then python .github/scripts/story_site.py jp-check; fi python .github/scripts/music_data.py build "$WORK/out" + - name: Keep the chart statistics for the next build + uses: actions/cache/save@v6 + with: + path: ${{ env.MUSIC_DATA_STATS_CACHE }} + key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Gates id: gates run: python .github/scripts/music_data.py check "$WORK/out" "$WORK/master" "$WORK/page/examples/songs" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c19c4b..1fc2282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,24 +2,36 @@ All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/). -## Unreleased +## [0.1.4](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.3...v0.1.4) - 2026-10-06 ### Features - **deck-data:** Export character rank experience thresholds ([3abe101](https://github.com/MetaSekaiLab/nnnotes/commit/3abe1017e47d267f77822af949500789c1831ee4)) - **music-data:** Publish the recommendation engine in the replay manifest ([42098d7](https://github.com/MetaSekaiLab/nnnotes/commit/42098d7bfa538912bbc350beaebfe78f3b04f202)) +- **catalogs:** Record the resource version of an imported catalog ([a9d0bbd](https://github.com/MetaSekaiLab/nnnotes/commit/a9d0bbd034aee00c696979eba3b1f33754b8a237)) +- **music-data:** Take the deck model's WASM release packages as engines ([de34ee2](https://github.com/MetaSekaiLab/nnnotes/commit/de34ee28ec181cac16237aa2751ff38e3bccfd91)) ### Bug fixes - **deck-data:** Export represented VIP rank identities ([c5d0514](https://github.com/MetaSekaiLab/nnnotes/commit/c5d051454f90c2afc7fb0063485f7dcf5d5a7669)) +- **music-data:** Name the mismatch when an engine package is rejected ([e7fdacd](https://github.com/MetaSekaiLab/nnnotes/commit/e7fdacdd34b80fdf391614f08fd5d7480d33904e)) ### Documentation -- Add a changelog generated by git-cliff ([7e871c5](https://github.com/MetaSekaiLab/nnnotes/commit/7e871c5f4dda7bde9323e0a8a71fb11d3cabeba3)) +- Add a changelog generated by git-cliff ([d601820](https://github.com/MetaSekaiLab/nnnotes/commit/d60182018911d3f1fc236d355185592b2b6ec8ad)) ### Build - **deck:** Build the deck model from the ournotes-sim crate ([946fb33](https://github.com/MetaSekaiLab/nnnotes/commit/946fb335aef234837c461dfa960f3ce28dc946da)) +- **deck:** Ournotes-deck ddf212b601731f9ab74245119d4408d2746a370c ([9f0ba0e](https://github.com/MetaSekaiLab/nnnotes/commit/9f0ba0e234cc4119682b3be4ebcd85344e618b02)) +- **deck:** Ournotes-deck v0.0.2 ([8a088f4](https://github.com/MetaSekaiLab/nnnotes/commit/8a088f4b7e40a79b3bc111e88c25d0abce397f36)) + +### CI + +- **commitlint:** Lift the line length limit of bodies and footers ([b9eecd9](https://github.com/MetaSekaiLab/nnnotes/commit/b9eecd91c682011748a3877e311d6ddcfb88000b)) +- Test on Python 3.14 ([5c845eb](https://github.com/MetaSekaiLab/nnnotes/commit/5c845eb365fcc23c36bf724f336ed747ba198057)) +- **deck:** Move create-pull-request to v8, which runs on Node.js 24 ([1e1bb17](https://github.com/MetaSekaiLab/nnnotes/commit/1e1bb179f98dddecb72465949276cbc4fb4e8fd7)) +- **deck:** Dispatch CI and the commit message check on the pin pull request ([3a6bdfc](https://github.com/MetaSekaiLab/nnnotes/commit/3a6bdfc95dd80171724ac313e2aeb5278859a843)) ## [0.1.3](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.2...v0.1.3) - 2026-09-30 diff --git a/docs/commands.md b/docs/commands.md index c8e9b58..1280243 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -679,7 +679,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] - [--no-bgm] [--jackets DIR] -o FILE + [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings @@ -697,7 +697,9 @@ master data tables about cards, skills, bonuses, scores and events. Single-skill default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and `--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's `deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a -luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads +luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what +they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures +only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), `--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `.json` diff --git a/docs/music-data.md b/docs/music-data.md index 7884030..90dc74d 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -12,7 +12,7 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] - [--no-bgm] [--jackets DIR] -o FILE + [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` - `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists @@ -33,6 +33,11 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful - `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements. `--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples (default 64). Lower caps reduce work but may leave the standard-error target unmet. +- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what + they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude + options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured + again, so an export after a deck model release that keeps the model's sources, or after a master data update that + keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a @@ -439,7 +444,8 @@ release in `rust/Cargo.toml` `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in -every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM +every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`): +releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM packages. ## Versions diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index b1aff1d..44e3cdb 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -169,6 +169,11 @@ "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "sourceSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics" + }, "format": { "type": "string", "description": "the statistics format, ournotes-deck.chart-stats/2" diff --git a/release-notes/v0.1.4.txt b/release-notes/v0.1.4.txt new file mode 100644 index 0000000..015ef3b --- /dev/null +++ b/release-notes/v0.1.4.txt @@ -0,0 +1,9 @@ +nnnotes 0.1.4 builds the deck model from ournotes-deck 0.0.2 and takes the deck model's WASM release packages as the replay and recommendation engines of the music data. + +- The bundled deck model `nnnotes._deck` is built from the `ournotes-sim` crate of ournotes-deck, pinned to its 0.0.2 release (commit `76e96b6`). Music data made with this version names that commit in `provenance.deck.commit`; its deck statistics can differ from files made with 0.1.3. +- `nnnotes music-data --replay-dir DIR --replay-engine PKG --recommend-engine PKG` takes the WASM release packages of the pinned ournotes-deck release, `ournotes-replay-wasm-v.tar.gz` and `ournotes-recommend-wasm-v.tar.gz`, as the archive or its unpacked directory. Each package's `build-info.json` must name its module and the pinned model commit, and the SHA-256 of its web JS and WASM files. The replay bundle carries the web JS, the WASM and `build-info.json` of both packages, listed as `engine` and `recommendEngine` in the replay manifest; a rejected package names the mismatch. See the [replay format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/replay.md). +- The deck model input (`music-data --full` and the replay bundle's `deck-data.json`) adds `MasterVip` (`_id`, `_vipRank`), so every VIP rank is present even without a bonus row, and `MasterCharacterRank._exp`, the cumulative experience each character rank needs. +- `nnnotes catalogs import` records the resource version without `--resource-version`: a JP catalog's source version, or the version in an international `catalog__.bin` file name. The version is also the default label. An invalid `--resource-version`, or one that differs from a JP catalog's source version, is an error. +- `CHANGELOG.md` lists every release and ships in the source distribution. + +Validation: Python 3.11, 3.12, 3.13 and 3.14 CI, strict golden records, Rust lints and the commit message check pass with the new deck model pin. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 78763a6..3146643 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -80,7 +80,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "nnnotes-deck" -version = "0.1.3" +version = "0.1.4" dependencies = [ "ournotes-sim", "pyo3", @@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" -version = "0.0.2" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=76e96b617842bdcc031cce98e3dfeead27ddbb85#76e96b617842bdcc031cce98e3dfeead27ddbb85" +version = "0.0.3" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e#31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 44aca0f..54ef161 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "nnnotes-deck" # the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here -version = "0.1.3" +version = "0.1.4" edition = "2024" rust-version = "1.85" description = "nnnotes._deck: the ournotes-sim chart statistics as a Python extension module" @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4849505..aec368c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -95,7 +95,8 @@ fn chart_stats( } /// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit: -/// its crate ournotes-sim as built) and the formats it reads and writes. +/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the +/// same model) and the formats it reads and writes. #[pyfunction] fn info(py: Python<'_>) -> PyResult> { let d = PyDict::new(py); @@ -103,6 +104,7 @@ fn info(py: Python<'_>) -> PyResult> { d.set_item("version", env!("DECK_VERSION"))?; d.set_item("source", env!("DECK_SOURCE"))?; d.set_item("commit", env!("DECK_COMMIT"))?; + d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?; d.set_item("dataFormat", ournotes_sim::data::FORMAT)?; d.set_item("format", chartstats::FORMAT)?; Ok(d) diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index 25fa675..4a174d1 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -4,4 +4,4 @@ models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server addresses are supplied by the user through the configuration; nothing of the game is included. """ -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 14cc82b..cf4f083 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -37,7 +37,8 @@ [--font emoji=] [--region [--region ...] | --all-regions] nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] - [--no-gekisou-aptitude] [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz] + [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR] + -o out/music-data.json[.gz] nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] nnnotes run-stage [...] @@ -678,7 +679,7 @@ def cmd_music_data(args, cfg): deck = None if args.no_deck else musicdata.Deck( seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds, - require_convergence=not args.allow_unconverged_aptitude) + require_convergence=not args.allow_unconverged_aptitude, cache=args.stats_cache) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED elif args.decoded_master: # decoded elsewhere: no master key @@ -998,6 +999,9 @@ def target(m, what): help="diagnostic export only: retain unmet SE flags at the sample cap; final exports reject them") c.add_argument("--aptitude-cross-seeds", type=int, metavar="N", help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)") + c.add_argument("--stats-cache", type=Path, metavar="DIR", + help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables " + "and chart are unchanged is not measured again; DIR then holds this export's charts only") c.add_argument("--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--replay-dir", metavar="DIR", diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index 0b58978..9c8be53 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -306,11 +306,13 @@ class Deck: """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None: - the deck model's defaults).""" + the deck model's defaults). With a `cache` directory, a chart's statistics are kept there under the SHA-256 of + what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and + the chart; a chart found there is not measured again.""" def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True, aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None, - require_convergence: bool = True): + require_convergence: bool = True, cache: Path | None = None): if module is None: try: from . import _deck as module @@ -321,14 +323,15 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \ aptitude, aptitude_max_seeds, aptitude_cross_seeds self.require_convergence = require_convergence + self.cache = None if cache is None else Path(cache) + self.counts: dict | None = None # {cached, measured} charts of the last stats() def info(self) -> dict: - """{name, version, source, commit, format} of the deck model.""" + """{name, version, source, commit, sourceSha256, format} of the deck model.""" i = self.module.info() - return {k: i[k] for k in ("name", "version", "source", "commit", "format")} + return {k: i[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} - def stats(self, deck_input: dict) -> dict: - """The chart statistics document of a deck input document (deckdata.build), its numbers as written.""" + def _measure(self, deck_input: dict) -> dict: try: text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers, aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds, @@ -338,6 +341,52 @@ def stats(self, deck_input: dict) -> dict: doc = json.loads(text, parse_float=deckdata._Num) if doc.get("format") != self.info()["format"]: raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") + return doc + + def cache_keys(self, deck_input: dict) -> list[str]: + """The cache key of each chart of a deck input document, in chart order.""" + info = self.info() + base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds, + "aptitude": self.aptitude, "aptitudeMaxSeeds": self.aptitude_max_seeds, + "aptitudeCrossSeeds": self.aptitude_cross_seeds, + "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()} + return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()})) + .hexdigest() for c in deck_input["charts"]] + + def _cached_stats(self, deck_input: dict) -> dict: + """stats() through the cache: measure the charts it lacks, add them, then keep only this document's charts.""" + from .cache import write_atomic + keys = self.cache_keys(deck_input) + found = {} + for k in keys: + path = self.cache / f"{k}.json" + if path.is_file(): + found[k] = json.loads(path.read_bytes(), parse_float=deckdata._Num) + missing = [c for c, k in zip(deck_input["charts"], keys) if k not in found] + doc = self._measure({**deck_input, "charts": missing}) + if len(doc.get("charts") or []) != len(missing): + raise MusicDataError("deck model: the charts measured differ from the charts asked for") + self.cache.mkdir(parents=True, exist_ok=True) + measured = iter(doc["charts"]) + for k in keys: + if k not in found: + found[k] = next(measured) + write_atomic(self.cache / f"{k}.json", deckdata.encode(found[k])) + for path in self.cache.glob("*.json"): + if path.stem not in found: + path.unlink() + doc["charts"] = [found[k] for k in keys] + self.counts = {"cached": len(keys) - len(missing), "measured": len(missing)} + return doc + + def stats(self, deck_input: dict) -> dict: + """The chart statistics document of a deck input document (deckdata.build), its numbers as written. With a + cache, the cache then holds exactly this document's charts.""" + if self.cache is None: + doc = self._measure(deck_input) + self.counts = {"cached": 0, "measured": len(deck_input["charts"])} + else: + doc = self._cached_stats(deck_input) if self.require_convergence and self.aptitude: unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds")) for c in doc.get("charts", []) @@ -1141,6 +1190,7 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source, "masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts), "deck": doc["provenance"]["deck"]["commit"] if deck is not None else None, + **({"deckStats": deck.counts} if deck is not None else {}), "unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]), "full": full, "bgm": bgm is not None, "jackets": len(jackets), "bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest(), diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 54b4aed..db077db 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -260,6 +260,7 @@ class FakeDeck: """The interface of nnnotes._deck: statistics made from the deck input as the model reports them, with the Gekisou aptitude (SHAPES; `aptitude`) unless it is left out.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" + SOURCE = "5" * 64 FORMAT = "ournotes-deck.chart-stats/2" def __init__(self, change=None, fail=None, header=None): @@ -267,7 +268,8 @@ def __init__(self, change=None, fail=None, header=None): def info(self): return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck", - "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT} + "commit": self.COMMIT, "sourceSha256": self.SOURCE, "dataFormat": deckdata.DECK_FORMAT, + "format": self.FORMAT} def chart_stats(self, data, seeds, workers, aptitude=True, aptitude_max_seeds=None, aptitude_cross_seeds=None): if self.fail: @@ -343,6 +345,44 @@ def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw) +def test_the_stats_cache_measures_only_charts_it_lacks(tmp_path): + cache = tmp_path / "stats" + + def run(name, deck, seeds=musicdata.DECK_SEEDS, stats_cache=cache): + (tmp_path / name).mkdir() + d = master_dir(tmp_path / name) + return musicdata.export(tmp_path / name / "music.json", deckdata.master_files(d), KEY, CHARTS.__getitem__, + bgm, **PROV, deck=musicdata.Deck(module=deck, seeds=seeds, cache=stats_cache)) + plain = run("plain", FakeDeck(), stats_cache=None) + fake = FakeDeck() + first = run("first", fake) + n = plain["charts"] + assert len(fake.inputs[0][0]["charts"]) == n and first["deckStats"] == {"cached": 0, "measured": n} + assert plain["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n + again = run("again", fake) + assert fake.inputs[1][0]["charts"] == [] and again["deckStats"] == {"cached": n, "measured": 0} + assert plain["sha256"] == first["sha256"] == again["sha256"] + # the key names the model's sources and the options: changing either measures every chart again, and the + # cache then holds the new statistics only + other = FakeDeck() + other.SOURCE = "6" * 64 + assert run("other", other)["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n + assert run("seeds", FakeDeck(), seeds=4)["deckStats"] == {"cached": 0, "measured": n} + + +def test_the_stats_cache_key_reads_the_master_tables_and_the_chart(): + deck = musicdata.Deck(module=FakeDeck()) + doc = {"master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[1]]}}, + "charts": [{"scoreId": 1, "notes": [1]}, {"scoreId": 2, "notes": [2]}]} + keys = deck.cache_keys(doc) + assert len(set(keys)) == 2 and all(re.fullmatch(r"[0-9a-f]{64}", k) for k in keys) + assert deck.cache_keys({**doc, "charts": doc["charts"][1:]}) == keys[1:] + changed = {**doc, "charts": [doc["charts"][0], {"scoreId": 2, "notes": [3]}]} + assert deck.cache_keys(changed)[0] == keys[0] and deck.cache_keys(changed)[1] != keys[1] + other = {**doc, "master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[2]]}}} + assert not set(deck.cache_keys(other)) & set(keys) + + def schema_validator(): jsonschema = pytest.importorskip("jsonschema") doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8")) @@ -435,7 +475,7 @@ def test_deck(tmp_path): assert [c["scoreId"] for c in deck_input["charts"]] == [10, 20, 30] # the songs' charts, not chart 40 assert deck_input["provenance"]["master"] == {"source": "api", "version": "v-test"} p = doc["provenance"] - assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")} + assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False)) assert doc["deck"]["model"]["power"] == 300000 assert doc["deck"]["gekisouAptitude"]["shapes"] == SHAPES @@ -630,6 +670,7 @@ def test_deck_module(): info = deck.info() assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2" assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) + assert re.fullmatch(r"[0-9a-f]{64}", info["sourceSha256"]) lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins with pytest.raises(ValueError, match="not a deck data file"):