Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/MUSIC_DATA.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ extend the existing linear plain-skill UI domain or derive source rank threshold
`ournotes-sim` package `rust/Cargo.lock` pins, each checked against the release's `SHA256SUMS`; nnnotes takes
their web packages after checking each `build-info.json` (module, commit, file SHA-256) against its deck model;
- `nnnotes music-data --decoded-master --jackets jackets --replay-dir replay --replay-engine ... --recommend-engine
... -o music-data.json`: the master data as decoded (no
... --stats-cache $MUSIC_DATA_STATS_CACHE -o music-data.json`: the deck statistics of the last build of the
region come back from `actions/cache` (derived numbers the file publishes, no game files), and only the charts
whose deck model sources (`provenance.deck.sourceSha256`), options, master tables or chart changed are measured
again; the summary counts both. The master data as decoded (no
master key), `provenance.master` the manifest's version and SHA-256 of the files as served; the charts, cue
sheets and jackets from the TW catalog, downloaded afresh on every run (never `actions/cache`: nnnotes keeps a
downloaded catalog for good, and the cache holds decrypted game files);
Expand Down
16 changes: 11 additions & 5 deletions .github/scripts/music_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
index.json, MasterManifest.json included) and its index entry as OUT.snapshot.json
build OUT `nnnotes music-data --decoded-master --jackets OUT/jackets -o OUT/music-data.json` ([paths]
master: the master step's OUT) with the replay and recommendation engines of the pinned
ournotes-deck release, its printed summary in OUT/music-data.summary.json
ournotes-deck release and the chart statistics cache $MUSIC_DATA_STATS_CACHE, its
printed summary in OUT/music-data.summary.json
check OUT MASTER PAGE the quality gates (.github/MUSIC_DATA.md) on OUT/music-data.json, with the published file
as the baseline and PAGE the chart data page's modules (examples/songs) for the smoke test;
the report in OUT/check.json, and OUT/build.json (the build marker) when every gate passed;
Expand Down Expand Up @@ -342,15 +343,18 @@ def cmd_build(out: str) -> None:
o.mkdir(parents=True, exist_ok=True)
nnnotes = [sys.executable, "-m", "nnnotes"]
usage = subprocess.run(nnnotes + ["music-data", "--help"], capture_output=True, text=True).stdout
if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine")):
fail("the installed nnnotes lacks decoded-master/replay/recommend export (sync the fork with upstream)")
if any(flag not in usage for flag in ("--decoded-master", "--replay-dir", "--replay-engine", "--recommend-engine",
"--stats-cache")):
fail("the installed nnnotes lacks decoded-master/replay/recommend export or the statistics cache (sync the "
"fork with upstream)")
engine, recommend = release_engines(o)
if env("MASTERDATA_REGION") == "jp":
import jp_vpngate
jp_vpngate.start()
cmd = nnnotes + ["music-data", "--decoded-master", "--jackets", str(o / "jackets"),
"--replay-dir", str(o / "replay"), "--replay-engine", str(engine),
"--recommend-engine", str(recommend), "-o", str(o / FILE)]
"--recommend-engine", str(recommend), "--stats-cache", env("MUSIC_DATA_STATS_CACHE"),
"-o", str(o / FILE)]
print("+ " + " ".join(cmd[1:]), flush=True)
with open(o / "music-data.summary.json", "wb") as f:
status = subprocess.run(cmd, stdout=f).returncode
Expand All @@ -360,8 +364,10 @@ def cmd_build(out: str) -> None:
import brotli
(o / FILE_BR).write_bytes(brotli.compress(raw, quality=9))
r = json.loads((o / "music-data.summary.json").read_text(encoding="utf-8"))
stats = r.get("deckStats") or {}
summary(f"- built: {r.get('songs')} songs, {r.get('charts')} charts, {r.get('jackets')} jackets, deck "
f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}")
f"{short(r.get('deck'))}, {r.get('bytes')} bytes, sha256 {short(r.get('sha256'))}; chart statistics "
f"{stats.get('cached')} cached, {stats.get('measured')} measured")


def release_engines(out: Path) -> tuple[Path, ...]:
Expand Down
17 changes: 17 additions & 0 deletions .github/workflows/music-data-region.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ env:
PYTHON_VERSION: "3.13"
PYTHONPATH: src
WORK: ${{ github.workspace }}/music-data-work
# The deck statistics of every chart of the last build, under the SHA-256 of the model sources, options, master
# tables and chart they come from (nnnotes music-data --stats-cache): derived numbers the file publishes, no game
# files. A build measures only the charts whose inputs changed.
MUSIC_DATA_STATS_CACHE: ${{ github.workspace }}/music-data-work/stats-cache
# Bundle keys come from secrets; catalog versions and CDN roots come from the saved master snapshot.
NNNOTES_CATALOG_REGION: ${{ inputs.region == 'jp' && 'jp' || 'tw' }}
NNNOTES_CATALOG_LANGUAGE: ${{ inputs.region == 'jp' && 'ja' || 'zh-Hant' }}
Expand Down Expand Up @@ -127,6 +131,13 @@ jobs:
- name: Master data
run: python .github/scripts/music_data.py master "$WORK/master"

- name: Chart statistics of the last build
uses: actions/cache/restore@v6
with:
path: ${{ env.MUSIC_DATA_STATS_CACHE }}
key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: music-data-stats-${{ inputs.region }}-

- name: Build
env:
NNNOTES_BUNDLE_KEY: ${{ secrets.NNNOTES_BUNDLE_KEY }}
Expand All @@ -140,6 +151,12 @@ jobs:
if [ "${{ inputs.region }}" = "jp" ]; then python .github/scripts/story_site.py jp-check; fi
python .github/scripts/music_data.py build "$WORK/out"

- name: Keep the chart statistics for the next build
uses: actions/cache/save@v6
with:
path: ${{ env.MUSIC_DATA_STATS_CACHE }}
key: music-data-stats-${{ inputs.region }}-${{ github.run_id }}-${{ github.run_attempt }}

- name: Gates
id: gates
run: python .github/scripts/music_data.py check "$WORK/out" "$WORK/master" "$WORK/page/examples/songs"
Expand Down
16 changes: 14 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,36 @@

All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/).

## Unreleased
## [0.1.4](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.3...v0.1.4) - 2026-10-06

### Features

- **deck-data:** Export character rank experience thresholds ([3abe101](https://github.com/MetaSekaiLab/nnnotes/commit/3abe1017e47d267f77822af949500789c1831ee4))
- **music-data:** Publish the recommendation engine in the replay manifest ([42098d7](https://github.com/MetaSekaiLab/nnnotes/commit/42098d7bfa538912bbc350beaebfe78f3b04f202))
- **catalogs:** Record the resource version of an imported catalog ([a9d0bbd](https://github.com/MetaSekaiLab/nnnotes/commit/a9d0bbd034aee00c696979eba3b1f33754b8a237))
- **music-data:** Take the deck model's WASM release packages as engines ([de34ee2](https://github.com/MetaSekaiLab/nnnotes/commit/de34ee28ec181cac16237aa2751ff38e3bccfd91))

### Bug fixes

- **deck-data:** Export represented VIP rank identities ([c5d0514](https://github.com/MetaSekaiLab/nnnotes/commit/c5d051454f90c2afc7fb0063485f7dcf5d5a7669))
- **music-data:** Name the mismatch when an engine package is rejected ([e7fdacd](https://github.com/MetaSekaiLab/nnnotes/commit/e7fdacdd34b80fdf391614f08fd5d7480d33904e))

### Documentation

- Add a changelog generated by git-cliff ([7e871c5](https://github.com/MetaSekaiLab/nnnotes/commit/7e871c5f4dda7bde9323e0a8a71fb11d3cabeba3))
- Add a changelog generated by git-cliff ([d601820](https://github.com/MetaSekaiLab/nnnotes/commit/d60182018911d3f1fc236d355185592b2b6ec8ad))

### Build

- **deck:** Build the deck model from the ournotes-sim crate ([946fb33](https://github.com/MetaSekaiLab/nnnotes/commit/946fb335aef234837c461dfa960f3ce28dc946da))
- **deck:** Ournotes-deck ddf212b601731f9ab74245119d4408d2746a370c ([9f0ba0e](https://github.com/MetaSekaiLab/nnnotes/commit/9f0ba0e234cc4119682b3be4ebcd85344e618b02))
- **deck:** Ournotes-deck v0.0.2 ([8a088f4](https://github.com/MetaSekaiLab/nnnotes/commit/8a088f4b7e40a79b3bc111e88c25d0abce397f36))

### CI

- **commitlint:** Lift the line length limit of bodies and footers ([b9eecd9](https://github.com/MetaSekaiLab/nnnotes/commit/b9eecd91c682011748a3877e311d6ddcfb88000b))
- Test on Python 3.14 ([5c845eb](https://github.com/MetaSekaiLab/nnnotes/commit/5c845eb365fcc23c36bf724f336ed747ba198057))
- **deck:** Move create-pull-request to v8, which runs on Node.js 24 ([1e1bb17](https://github.com/MetaSekaiLab/nnnotes/commit/1e1bb179f98dddecb72465949276cbc4fb4e8fd7))
- **deck:** Dispatch CI and the commit message check on the pin pull request ([3a6bdfc](https://github.com/MetaSekaiLab/nnnotes/commit/3a6bdfc95dd80171724ac313e2aeb5278859a843))

## [0.1.3](https://github.com/MetaSekaiLab/nnnotes/compare/v0.1.2...v0.1.3) - 2026-09-30

Expand Down
6 changes: 4 additions & 2 deletions docs/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ
```
nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
[--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
[--no-bgm] [--jackets DIR] -o FILE
[--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
```

Writes one JSON file with every `MasterLiveMusic` song and its charts for one master data version: titles, readings
Expand All @@ -697,7 +697,9 @@ master data tables about cards, skills, bonuses, scores and events. Single-skill
default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and
`--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's
`deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a
luck range and its threads. The master data is decoded from the files as served: `--master-files DIR` reads
luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what
they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures
only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads
`DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`),
`--apk-master` the same files inside `[paths] apk` (region `embedded`); each file is checked against the manifest's
SHA-256. `--decoded-master` reads master data decoded elsewhere instead, without the master key: the `<Table>.json`
Expand Down
10 changes: 8 additions & 2 deletions docs/music-data.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; i
```
nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N]
[--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N]
[--no-bgm] [--jackets DIR] -o FILE
[--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE
```

- `--master-files DIR`: master data files as served, `DIR/MasterManifest.json` and the `.bin` files it lists
Expand All @@ -33,6 +33,11 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful
- `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements.
`--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples
(default 64). Lower caps reduce work but may leave the standard-error target unmet.
- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what
they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude
options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured
again, so an export after a deck model release that keeps the model's sources, or after a master data update that
keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only.
- `--no-bgm`: do not read the cue sheets (every `bgm.length` is null).
- `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/<jacket>`, as `DIR/<jacket>.webp`
(WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a
Expand Down Expand Up @@ -439,7 +444,8 @@ release in `rust/Cargo.toml`
`pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes
version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml`
opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in
every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`):
releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
packages.

## Versions
Expand Down
5 changes: 5 additions & 0 deletions docs/schema/music-data.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
"sourceSha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$",
"description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics"
},
"format": {
"type": "string",
"description": "the statistics format, ournotes-deck.chart-stats/2"
Expand Down
9 changes: 9 additions & 0 deletions release-notes/v0.1.4.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
nnnotes 0.1.4 builds the deck model from ournotes-deck 0.0.2 and takes the deck model's WASM release packages as the replay and recommendation engines of the music data.

- The bundled deck model `nnnotes._deck` is built from the `ournotes-sim` crate of ournotes-deck, pinned to its 0.0.2 release (commit `76e96b6`). Music data made with this version names that commit in `provenance.deck.commit`; its deck statistics can differ from files made with 0.1.3.
- `nnnotes music-data --replay-dir DIR --replay-engine PKG --recommend-engine PKG` takes the WASM release packages of the pinned ournotes-deck release, `ournotes-replay-wasm-v<version>.tar.gz` and `ournotes-recommend-wasm-v<version>.tar.gz`, as the archive or its unpacked directory. Each package's `build-info.json` must name its module and the pinned model commit, and the SHA-256 of its web JS and WASM files. The replay bundle carries the web JS, the WASM and `build-info.json` of both packages, listed as `engine` and `recommendEngine` in the replay manifest; a rejected package names the mismatch. See the [replay format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/replay.md).
- The deck model input (`music-data --full` and the replay bundle's `deck-data.json`) adds `MasterVip` (`_id`, `_vipRank`), so every VIP rank is present even without a bonus row, and `MasterCharacterRank._exp`, the cumulative experience each character rank needs.
- `nnnotes catalogs import` records the resource version without `--resource-version`: a JP catalog's source version, or the version in an international `catalog_<version>_<language>.bin` file name. The version is also the default label. An invalid `--resource-version`, or one that differs from a JP catalog's source version, is an error.
- `CHANGELOG.md` lists every release and ships in the source distribution.

Validation: Python 3.11, 3.12, 3.13 and 3.14 CI, strict golden records, Rust lints and the commit message check pass with the new deck model pin.
6 changes: 3 additions & 3 deletions rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions rust/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
name = "nnnotes-deck"
# the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here
version = "0.1.3"
version = "0.1.4"
edition = "2024"
rust-version = "1.85"
description = "nnnotes._deck: the ournotes-sim chart statistics as a Python extension module"
Expand All @@ -16,7 +16,7 @@ crate-type = ["cdylib"]
pyo3 = { version = "0.29", features = ["abi3-py311"] }
# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release;
# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release
ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" }
ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" }
serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] }

[profile.release]
Expand Down
4 changes: 3 additions & 1 deletion rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,16 @@ fn chart_stats(
}

/// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit:
/// its crate ournotes-sim as built) and the formats it reads and writes.
/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the
/// same model) and the formats it reads and writes.
#[pyfunction]
fn info(py: Python<'_>) -> PyResult<Bound<'_, PyDict>> {
let d = PyDict::new(py);
d.set_item("name", "ournotes-deck")?;
d.set_item("version", env!("DECK_VERSION"))?;
d.set_item("source", env!("DECK_SOURCE"))?;
d.set_item("commit", env!("DECK_COMMIT"))?;
d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?;
d.set_item("dataFormat", ournotes_sim::data::FORMAT)?;
d.set_item("format", chartstats::FORMAT)?;
Ok(d)
Expand Down
2 changes: 1 addition & 1 deletion src/nnnotes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server
addresses are supplied by the user through the configuration; nothing of the game is included.
"""
__version__ = "0.1.3"
__version__ = "0.1.4"
Loading
Loading