diff --git a/.github/MUSIC_DATA.md b/.github/MUSIC_DATA.md
index a4cfeff..89c5825 100644
--- a/.github/MUSIC_DATA.md
+++ b/.github/MUSIC_DATA.md
@@ -98,7 +98,10 @@ extend the existing linear plain-skill UI domain or derive source rank threshold
`ournotes-sim` package `rust/Cargo.lock` pins, each checked against the release's `SHA256SUMS`; nnnotes takes
their web packages after checking each `build-info.json` (module, commit, file SHA-256) against its deck model;
- `nnnotes music-data --decoded-master --jackets jackets --replay-dir replay --replay-engine ... --recommend-engine
- ... -o music-data.json`: the master data as decoded (no
+ ... --stats-cache $MUSIC_DATA_STATS_CACHE -o music-data.json`: the deck statistics of the last build of the
+ region come back from `actions/cache` (derived numbers the file publishes, no game files), and only the charts
+ whose deck model sources (`provenance.deck.sourceSha256`), options, master tables or chart changed are measured
+ again; the summary counts both. The master data as decoded (no
master key), `provenance.master` the manifest's version and SHA-256 of the files as served; the charts, cue
sheets and jackets from the TW catalog, downloaded afresh on every run (never `actions/cache`: nnnotes keeps a
downloaded catalog for good, and the cache holds decrypted game files);
diff --git a/.github/scripts/music_data.py b/.github/scripts/music_data.py
index 246fc40..5fe1561 100755
--- a/.github/scripts/music_data.py
+++ b/.github/scripts/music_data.py
@@ -11,7 +11,8 @@
index.json, MasterManifest.json included) and its index entry as OUT.snapshot.json
build OUT `nnnotes music-data --decoded-master --jackets OUT/jackets -o OUT/music-data.json` ([paths]
master: the master step's OUT) with the replay and recommendation engines of the pinned
- ournotes-deck release, its printed summary in OUT/music-data.summary.json
+ ournotes-deck release and the chart statistics cache $MUSIC_DATA_STATS_CACHE, its
+ printed summary in OUT/music-data.summary.json
check OUT MASTER PAGE the quality gates (.github/MUSIC_DATA.md) on OUT/music-data.json, with the published file
as the baseline and PAGE the chart data page's modules (examples/songs) for the smoke test;
the report in OUT/check.json, and OUT/build.json (the build marker) when every gate passed;
@@ -342,15 +343,18 @@ def cmd_build(out: str) -> None:
o.mkdir(parents=True, exist_ok=True)
nnnotes = [sys.executable, "-m", "nnnotes"]
usage = subprocess.run(nnnotes + ["music-data", "--help"], capture_output=True, text=True).stdout
- if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine")):
- fail("the installed nnnotes lacks decoded-master/replay/recommend export (sync the fork with upstream)")
+ if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine",
+ "--stats-cache")):
+ fail("the installed nnnotes lacks decoded-master/replay/recommend export or the statistics cache (sync the "
+ "fork with upstream)")
engine, recommend = release_engines(o)
if env("MASTERDATA_REGION") == "jp":
import jp_vpngate
jp_vpngate.start()
cmd = nnnotes + ["music-data", "--decoded-master", "--jackets", str(o / "jackets"),
"--replay-dir", str(o / "replay"), "--replay-engine", str(engine),
- "--recommend-engine", str(recommend), "-o", str(o / FILE)]
+ "--recommend-engine", str(recommend), "--stats-cache", env("MUSIC_DATA_STATS_CACHE"),
+ "-o", str(o / FILE)]
print("+ " + " ".join(cmd[1:]), flush=True)
with open(o / "music-data.summary.json", "wb") as f:
status = subprocess.run(cmd, stdout=f).returncode
@@ -360,8 +364,10 @@ def cmd_build(out: str) -> None:
import brotli
(o / FILE_BR).write_bytes(brotli.compress(raw, quality=9))
r = json.loads((o / "music-data.summary.json").read_text(encoding="utf-8"))
+ stats = r.get("deckStats") or {}
summary(f"- built: {r.get('songs')} songs, {r.get('charts')} charts, {r.get('jackets')} jackets, deck "
- f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}")
+ f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}; chart statistics "
+ f"{stats.get('cached')} cached, {stats.get('measured')} measured")
def release_engines(out: Path) -> tuple[Path, ...]:
diff --git a/.github/workflows/music-data-region.yml b/.github/workflows/music-data-region.yml
index 611d107..292d23d 100644
--- a/.github/workflows/music-data-region.yml
+++ b/.github/workflows/music-data-region.yml
@@ -38,6 +38,10 @@ env:
PYTHON_VERSION: "3.13"
PYTHONPATH: src
WORK: ${{ github.workspace }}/music-data-work
+ # The deck statistics of every chart of the last build, under the SHA-256 of the model sources, options, master
+ # tables and chart they come from (nnnotes music-data --stats-cache): derived numbers the file publishes, no game
+ # files. A build measures only the charts whose inputs changed.
+ MUSIC_DATA_STATS_CACHE: ${{ github.workspace }}/music-data-work/stats-cache
# Bundle keys come from secrets; catalog versions and CDN roots come from the saved master snapshot.
NNNOTES_CATALOG_REGION: ${{ inputs.region == 'jp' && 'jp' || 'tw' }}
NNNOTES_CATALOG_LANGUAGE: ${{ inputs.region == 'jp' && 'ja' || 'zh-Hant' }}
@@ -127,6 +131,13 @@ jobs:
- name: Master data
run: python .github/scripts/music_data.py master "$WORK/master"
+ - name: Chart statistics of the last build
+ uses: actions/cache/restore@v6
+ with:
+ path: ${{ env.MUSIC_DATA_STATS_CACHE }}
+ key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }}
+ restore-keys: music-data-stats-${{ inputs.region }}-
+
- name: Build
env:
NNNOTES_BUNDLE_KEY: ${{ secrets.NNNOTES_BUNDLE_KEY }}
@@ -140,6 +151,12 @@ jobs:
if [ "${{ inputs.region }}" = "jp" ]; then python .github/scripts/story_site.py jp-check; fi
python .github/scripts/music_data.py build "$WORK/out"
+ - name: Keep the chart statistics for the next build
+ uses: actions/cache/save@v6
+ with:
+ path: ${{ env.MUSIC_DATA_STATS_CACHE }}
+ key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }}
+
- name: Gates
id: gates
run: python .github/scripts/music_data.py check "$WORK/out" "$WORK/master" "$WORK/page/examples/songs"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3c19c4b..1fc2282 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,24 +2,36 @@
All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/).
-## Unreleased
+## [0.1.4](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.3...v0.1.4) - 2026-10-06
### Features
- **deck-data:** Export character rank experience thresholds ([3abe101](https://github.com/MetaSekaiLab/nnnotes/commit/3abe1017e47d267f77822af949500789c1831ee4))
- **music-data:** Publish the recommendation engine in the replay manifest ([42098d7](https://github.com/MetaSekaiLab/nnnotes/commit/42098d7bfa538912bbc350beaebfe78f3b04f202))
+- **catalogs:** Record the resource version of an imported catalog ([a9d0bbd](https://github.com/MetaSekaiLab/nnnotes/commit/a9d0bbd034aee00c696979eba3b1f33754b8a237))
+- **music-data:** Take the deck model's WASM release packages as engines ([de34ee2](https://github.com/MetaSekaiLab/nnnotes/commit/de34ee28ec181cac16237aa2751ff38e3bccfd91))
### Bug fixes
- **deck-data:** Export represented VIP rank identities ([c5d0514](https://github.com/MetaSekaiLab/nnnotes/commit/c5d051454f90c2afc7fb0063485f7dcf5d5a7669))
+- **music-data:** Name the mismatch when an engine package is rejected ([e7fdacd](https://github.com/MetaSekaiLab/nnnotes/commit/e7fdacdd34b80fdf391614f08fd5d7480d33904e))
### Documentation
-- Add a changelog generated by git-cliff ([7e871c5](https://github.com/MetaSekaiLab/nnnotes/commit/7e871c5f4dda7bde9323e0a8a71fb11d3cabeba3))
+- Add a changelog generated by git-cliff ([d601820](https://github.com/MetaSekaiLab/nnnotes/commit/d60182018911d3f1fc236d355185592b2b6ec8ad))
### Build
- **deck:** Build the deck model from the ournotes-sim crate ([946fb33](https://github.com/MetaSekaiLab/nnnotes/commit/946fb335aef234837c461dfa960f3ce28dc946da))
+- **deck:** Ournotes-deck ddf212b601731f9ab74245119d4408d2746a370c ([9f0ba0e](https://github.com/MetaSekaiLab/nnnotes/commit/9f0ba0e234cc4119682b3be4ebcd85344e618b02))
+- **deck:** Ournotes-deck v0.0.2 ([8a088f4](https://github.com/MetaSekaiLab/nnnotes/commit/8a088f4b7e40a79b3bc111e88c25d0abce397f36))
+
+### CI
+
+- **commitlint:** Lift the line length limit of bodies and footers ([b9eecd9](https://github.com/MetaSekaiLab/nnnotes/commit/b9eecd91c682011748a3877e311d6ddcfb88000b))
+- Test on Python 3.14 ([5c845eb](https://github.com/MetaSekaiLab/nnnotes/commit/5c845eb365fcc23c36bf724f336ed747ba198057))
+- **deck:** Move create-pull-request to v8, which runs on Node.js 24 ([1e1bb17](https://github.com/MetaSekaiLab/nnnotes/commit/1e1bb179f98dddecb72465949276cbc4fb4e8fd7))
+- **deck:** Dispatch CI and the commit message check on the pin pull request ([3a6bdfc](https://github.com/MetaSekaiLab/nnnotes/commit/3a6bdfc95dd80171724ac313e2aeb5278859a843))
## [0.1.3](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.2...v0.1.3) - 2026-09-30
diff --git a/docs/commands.md b/docs/commands.md
index c8e9b58..1280243 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -679,7 +679,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ
```
nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
[--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
- [--no-bgm] [--jackets DIR] -o FILE
+ [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
```
Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings
@@ -697,7 +697,9 @@ master data tables about cards, skills, bonuses, scores and events. Single-skill
default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and
`--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's
`deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a
-luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads
+luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what
+they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures
+only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads
`DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`),
`--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's
SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `
.json`
diff --git a/docs/music-data.md b/docs/music-data.md
index 7884030..90dc74d 100644
--- a/docs/music-data.md
+++ b/docs/music-data.md
@@ -12,7 +12,7 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i
```
nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
[--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
- [--no-bgm] [--jackets DIR] -o FILE
+ [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
```
- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists
@@ -33,6 +33,11 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful
- `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements.
`--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples
(default 64). Lower caps reduce work but may leave the standard-error target unmet.
+- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what
+ they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude
+ options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured
+ again, so an export after a deck model release that keeps the model's sources, or after a master data update that
+ keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only.
- `--no-bgm`: do not read the cue sheets (every `bgm.length` is null).
- `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp`
(WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a
@@ -439,7 +444,8 @@ release in `rust/Cargo.toml`
`pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes
version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml`
opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in
-every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
+every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`):
+releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
packages.
## Versions
diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json
index b1aff1d..44e3cdb 100644
--- a/docs/schema/music-data.schema.json
+++ b/docs/schema/music-data.schema.json
@@ -169,6 +169,11 @@
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
+ "sourceSha256": {
+ "type": "string",
+ "pattern": "^[0-9a-f]{64}$",
+ "description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics"
+ },
"format": {
"type": "string",
"description": "the statistics format, ournotes-deck.chart-stats/2"
diff --git a/release-notes/v0.1.4.txt b/release-notes/v0.1.4.txt
new file mode 100644
index 0000000..015ef3b
--- /dev/null
+++ b/release-notes/v0.1.4.txt
@@ -0,0 +1,9 @@
+nnnotes 0.1.4 builds the deck model from ournotes-deck 0.0.2 and takes the deck model's WASM release packages as the replay and recommendation engines of the music data.
+
+- The bundled deck model `nnnotes._deck` is built from the `ournotes-sim` crate of ournotes-deck, pinned to its 0.0.2 release (commit `76e96b6`). Music data made with this version names that commit in `provenance.deck.commit`; its deck statistics can differ from files made with 0.1.3.
+- `nnnotes music-data --replay-dir DIR --replay-engine PKG --recommend-engine PKG` takes the WASM release packages of the pinned ournotes-deck release, `ournotes-replay-wasm-v.tar.gz` and `ournotes-recommend-wasm-v.tar.gz`, as the archive or its unpacked directory. Each package's `build-info.json` must name its module and the pinned model commit, and the SHA-256 of its web JS and WASM files. The replay bundle carries the web JS, the WASM and `build-info.json` of both packages, listed as `engine` and `recommendEngine` in the replay manifest; a rejected package names the mismatch. See the [replay format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/replay.md).
+- The deck model input (`music-data --full` and the replay bundle's `deck-data.json`) adds `MasterVip` (`_id`, `_vipRank`), so every VIP rank is present even without a bonus row, and `MasterCharacterRank._exp`, the cumulative experience each character rank needs.
+- `nnnotes catalogs import` records the resource version without `--resource-version`: a JP catalog's source version, or the version in an international `catalog__.bin` file name. The version is also the default label. An invalid `--resource-version`, or one that differs from a JP catalog's source version, is an error.
+- `CHANGELOG.md` lists every release and ships in the source distribution.
+
+Validation: Python 3.11, 3.12, 3.13 and 3.14 CI, strict golden records, Rust lints and the commit message check pass with the new deck model pin.
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 78763a6..3146643 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -80,7 +80,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "nnnotes-deck"
-version = "0.1.3"
+version = "0.1.4"
dependencies = [
"ournotes-sim",
"pyo3",
@@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "ournotes-sim"
-version = "0.0.2"
-source = "git+https://github.com/empty-sekai/ournotes-deck?rev=76e96b617842bdcc031cce98e3dfeead27ddbb85#76e96b617842bdcc031cce98e3dfeead27ddbb85"
+version = "0.0.3"
+source = "git+https://github.com/empty-sekai/ournotes-deck?rev=31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e#31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e"
dependencies = [
"serde",
"serde_json",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 44aca0f..54ef161 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "nnnotes-deck"
# the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here
-version = "0.1.3"
+version = "0.1.4"
edition = "2024"
rust-version = "1.85"
description = "nnnotes._deck: the ournotes-sim chart statistics as a Python extension module"
@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
pyo3 = { version = "0.29", features = ["abi3-py311"] }
# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release;
# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release
-ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" }
+ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" }
serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] }
[profile.release]
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
index 4849505..aec368c 100644
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -95,7 +95,8 @@ fn chart_stats(
}
/// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit:
-/// its crate ournotes-sim as built) and the formats it reads and writes.
+/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the
+/// same model) and the formats it reads and writes.
#[pyfunction]
fn info(py: Python<'_>) -> PyResult> {
let d = PyDict::new(py);
@@ -103,6 +104,7 @@ fn info(py: Python<'_>) -> PyResult> {
d.set_item("version", env!("DECK_VERSION"))?;
d.set_item("source", env!("DECK_SOURCE"))?;
d.set_item("commit", env!("DECK_COMMIT"))?;
+ d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?;
d.set_item("dataFormat", ournotes_sim::data::FORMAT)?;
d.set_item("format", chartstats::FORMAT)?;
Ok(d)
diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py
index 25fa675..4a174d1 100644
--- a/src/nnnotes/__init__.py
+++ b/src/nnnotes/__init__.py
@@ -4,4 +4,4 @@
models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server
addresses are supplied by the user through the configuration; nothing of the game is included.
"""
-__version__ = "0.1.3"
+__version__ = "0.1.4"
diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py
index 14cc82b..cf4f083 100644
--- a/src/nnnotes/cli.py
+++ b/src/nnnotes/cli.py
@@ -37,7 +37,8 @@
[--font emoji=]
[--region [--region ...] | --all-regions]
nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck]
- [--no-gekisou-aptitude] [--no-bgm] [--jackets DIR] -o out/music-data.json[.gz]
+ [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR]
+ -o out/music-data.json[.gz]
nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas]
nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ]
nnnotes run-stage [...]
@@ -678,7 +679,7 @@ def cmd_music_data(args, cfg):
deck = None if args.no_deck else musicdata.Deck(
seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude,
aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds,
- require_convergence=not args.allow_unconverged_aptitude)
+ require_convergence=not args.allow_unconverged_aptitude, cache=args.stats_cache)
if args.apk_master:
src, region = deckdata.apk_master(apk), deckdata.EMBEDDED
elif args.decoded_master: # decoded elsewhere: no master key
@@ -998,6 +999,9 @@ def target(m, what):
help="diagnostic export only: retain unmet SE flags at the sample cap; final exports reject them")
c.add_argument("--aptitude-cross-seeds", type=int, metavar="N",
help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)")
+ c.add_argument("--stats-cache", type=Path, metavar="DIR",
+ help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables "
+ "and chart are unchanged is not measured again; DIR then holds this export's charts only")
c.add_argument("--no-bgm", action="store_true",
help="do not read the BGM cue sheets (every song's bgm.length is null)")
c.add_argument("--replay-dir", metavar="DIR",
diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py
index 0b58978..9c8be53 100644
--- a/src/nnnotes/musicdata.py
+++ b/src/nnnotes/musicdata.py
@@ -306,11 +306,13 @@ class Deck:
"""The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the
available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude
too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None:
- the deck model's defaults)."""
+ the deck model's defaults). With a `cache` directory, a chart's statistics are kept there under the SHA-256 of
+ what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and
+ the chart; a chart found there is not measured again."""
def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True,
aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None,
- require_convergence: bool = True):
+ require_convergence: bool = True, cache: Path | None = None):
if module is None:
try:
from . import _deck as module
@@ -321,14 +323,15 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N
self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \
aptitude, aptitude_max_seeds, aptitude_cross_seeds
self.require_convergence = require_convergence
+ self.cache = None if cache is None else Path(cache)
+ self.counts: dict | None = None # {cached, measured} charts of the last stats()
def info(self) -> dict:
- """{name, version, source, commit, format} of the deck model."""
+ """{name, version, source, commit, sourceSha256, format} of the deck model."""
i = self.module.info()
- return {k: i[k] for k in ("name", "version", "source", "commit", "format")}
+ return {k: i[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")}
- def stats(self, deck_input: dict) -> dict:
- """The chart statistics document of a deck input document (deckdata.build), its numbers as written."""
+ def _measure(self, deck_input: dict) -> dict:
try:
text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers,
aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds,
@@ -338,6 +341,52 @@ def stats(self, deck_input: dict) -> dict:
doc = json.loads(text, parse_float=deckdata._Num)
if doc.get("format") != self.info()["format"]:
raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}")
+ return doc
+
+ def cache_keys(self, deck_input: dict) -> list[str]:
+ """The cache key of each chart of a deck input document, in chart order."""
+ info = self.info()
+ base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds,
+ "aptitude": self.aptitude, "aptitudeMaxSeeds": self.aptitude_max_seeds,
+ "aptitudeCrossSeeds": self.aptitude_cross_seeds,
+ "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()}
+ return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()}))
+ .hexdigest() for c in deck_input["charts"]]
+
+ def _cached_stats(self, deck_input: dict) -> dict:
+ """stats() through the cache: measure the charts it lacks, add them, then keep only this document's charts."""
+ from .cache import write_atomic
+ keys = self.cache_keys(deck_input)
+ found = {}
+ for k in keys:
+ path = self.cache / f"{k}.json"
+ if path.is_file():
+ found[k] = json.loads(path.read_bytes(), parse_float=deckdata._Num)
+ missing = [c for c, k in zip(deck_input["charts"], keys) if k not in found]
+ doc = self._measure({**deck_input, "charts": missing})
+ if len(doc.get("charts") or []) != len(missing):
+ raise MusicDataError("deck model: the charts measured differ from the charts asked for")
+ self.cache.mkdir(parents=True, exist_ok=True)
+ measured = iter(doc["charts"])
+ for k in keys:
+ if k not in found:
+ found[k] = next(measured)
+ write_atomic(self.cache / f"{k}.json", deckdata.encode(found[k]))
+ for path in self.cache.glob("*.json"):
+ if path.stem not in found:
+ path.unlink()
+ doc["charts"] = [found[k] for k in keys]
+ self.counts = {"cached": len(keys) - len(missing), "measured": len(missing)}
+ return doc
+
+ def stats(self, deck_input: dict) -> dict:
+ """The chart statistics document of a deck input document (deckdata.build), its numbers as written. With a
+ cache, the cache then holds exactly this document's charts."""
+ if self.cache is None:
+ doc = self._measure(deck_input)
+ self.counts = {"cached": 0, "measured": len(deck_input["charts"])}
+ else:
+ doc = self._cached_stats(deck_input)
if self.require_convergence and self.aptitude:
unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds"))
for c in doc.get("charts", [])
@@ -1141,6 +1190,7 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes],
return {"out": str(out), "format": FORMAT, "region": region, "masterSource": src.source,
"masterVersion": src.version, "songs": len(doc["songs"]), "charts": len(charts),
"deck": doc["provenance"]["deck"]["commit"] if deck is not None else None,
+ **({"deckStats": deck.counts} if deck is not None else {}),
"unplayable": sum(1 for c in charts if c["deck"] and c["deck"]["unplayable"]),
"full": full, "bgm": bgm is not None, "jackets": len(jackets),
"bytes": len(data), "fileBytes": len(written), "sha256": hashlib.sha256(written).hexdigest(),
diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py
index 54b4aed..db077db 100644
--- a/tests/test_musicdata.py
+++ b/tests/test_musicdata.py
@@ -260,6 +260,7 @@ class FakeDeck:
"""The interface of nnnotes._deck: statistics made from the deck input as the model reports them, with the Gekisou
aptitude (SHAPES; `aptitude`) unless it is left out."""
COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6"
+ SOURCE = "5" * 64
FORMAT = "ournotes-deck.chart-stats/2"
def __init__(self, change=None, fail=None, header=None):
@@ -267,7 +268,8 @@ def __init__(self, change=None, fail=None, header=None):
def info(self):
return {"name": "ournotes-deck", "version": "0.0.1", "source": "https://github.com/empty-sekai/ournotes-deck",
- "commit": self.COMMIT, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT}
+ "commit": self.COMMIT, "sourceSha256": self.SOURCE, "dataFormat": deckdata.DECK_FORMAT,
+ "format": self.FORMAT}
def chart_stats(self, data, seeds, workers, aptitude=True, aptitude_max_seeds=None, aptitude_cross_seeds=None):
if self.fail:
@@ -343,6 +345,44 @@ def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json",
deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw)
+def test_the_stats_cache_measures_only_charts_it_lacks(tmp_path):
+ cache = tmp_path / "stats"
+
+ def run(name, deck, seeds=musicdata.DECK_SEEDS, stats_cache=cache):
+ (tmp_path / name).mkdir()
+ d = master_dir(tmp_path / name)
+ return musicdata.export(tmp_path / name / "music.json", deckdata.master_files(d), KEY, CHARTS.__getitem__,
+ bgm, **PROV, deck=musicdata.Deck(module=deck, seeds=seeds, cache=stats_cache))
+ plain = run("plain", FakeDeck(), stats_cache=None)
+ fake = FakeDeck()
+ first = run("first", fake)
+ n = plain["charts"]
+ assert len(fake.inputs[0][0]["charts"]) == n and first["deckStats"] == {"cached": 0, "measured": n}
+ assert plain["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n
+ again = run("again", fake)
+ assert fake.inputs[1][0]["charts"] == [] and again["deckStats"] == {"cached": n, "measured": 0}
+ assert plain["sha256"] == first["sha256"] == again["sha256"]
+ # the key names the model's sources and the options: changing either measures every chart again, and the
+ # cache then holds the new statistics only
+ other = FakeDeck()
+ other.SOURCE = "6" * 64
+ assert run("other", other)["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n
+ assert run("seeds", FakeDeck(), seeds=4)["deckStats"] == {"cached": 0, "measured": n}
+
+
+def test_the_stats_cache_key_reads_the_master_tables_and_the_chart():
+ deck = musicdata.Deck(module=FakeDeck())
+ doc = {"master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[1]]}},
+ "charts": [{"scoreId": 1, "notes": [1]}, {"scoreId": 2, "notes": [2]}]}
+ keys = deck.cache_keys(doc)
+ assert len(set(keys)) == 2 and all(re.fullmatch(r"[0-9a-f]{64}", k) for k in keys)
+ assert deck.cache_keys({**doc, "charts": doc["charts"][1:]}) == keys[1:]
+ changed = {**doc, "charts": [doc["charts"][0], {"scoreId": 2, "notes": [3]}]}
+ assert deck.cache_keys(changed)[0] == keys[0] and deck.cache_keys(changed)[1] != keys[1]
+ other = {**doc, "master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[2]]}}}
+ assert not set(deck.cache_keys(other)) & set(keys)
+
+
def schema_validator():
jsonschema = pytest.importorskip("jsonschema")
doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8"))
@@ -435,7 +475,7 @@ def test_deck(tmp_path):
assert [c["scoreId"] for c in deck_input["charts"]] == [10, 20, 30] # the songs' charts, not chart 40
assert deck_input["provenance"]["master"] == {"source": "api", "version": "v-test"}
p = doc["provenance"]
- assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "format")}
+ assert p["deck"] == {k: fake.info()[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")}
assert list(p["master"]["tables"]) == list(musicdata.tables_of(True, False))
assert doc["deck"]["model"]["power"] == 300000
assert doc["deck"]["gekisouAptitude"]["shapes"] == SHAPES
@@ -630,6 +670,7 @@ def test_deck_module():
info = deck.info()
assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2"
assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"])
+ assert re.fullmatch(r"[0-9a-f]{64}", info["sourceSha256"])
lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8")
assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins
with pytest.raises(ValueError, match="not a deck data file"):