feat(data): load and validate split product snapshots

docs/data-format.md

@@ -90,7 +90,7 @@ A file entry has one of these forms:
   `.br`). `size` is the byte count of the file, `stored` that of the asset as stored (the encoded bytes). An asset is
   encoded only when that makes it smaller, so `stored` is less than `size`; `stored` is present exactly when the
   asset name ends in `.gz` or `.br`.
-- **Split JSON object** (a chart manifest's files only): `{ "parts": [[key, asset, size], ...], "size": <bytes> }`; a
+- **Split JSON object** (logical files and object JSON roles): `{ "parts": [[key, asset, size], ...], "size": <bytes> }`; a
   part whose asset is encoded is `[key, asset, size, stored]`. A large JSON file whose top level is an object is
   stored per top-level key: each part's asset holds the raw JSON text of that key's value. The file's text is rebuilt
   as
@@ -123,6 +123,17 @@ Several products may retain different file roles of the same object. Loading the
 roles; overlapping roles and object metadata must agree. Product paths may include region directories, and are
 relative to the layout root.
 
+### Immutable product manifests
+
+A site can store a product manifest at `products/<sha256>.json`, with the SHA-256 of the manifest bytes as its
+name. Its parent product's `parts` fields name that exact path, so each public entry point selects a coherent
+set of dependencies. Public aliases may remain at `lives/<id>.json`, `models/<id>.json` or `stories/<id>.json`.
+An alias in a region directory sets `root: "../../"`; a two-level path uses `"../"`. The live and story loaders
+honor that root, and a supplied `base` overrides it.
+
+The data validator checks immutable manifest contents against their names, then applies the product, subject,
+object and file checks. Ordinary product-layout manifest names continue to identify their subjects.
+
 An object is one Unity object, converted. Its id is `<serialized file>:<path id>` (the path id as decimal digits,
 possibly negative), the same in every product that uses it. An object's JSON names another object by a reference
 `{ "$ref": { "file": "<serialized file>", "pathId": "<path id>" } }`, the object `<file>:<pathId>`; `null` names none.

docs/story-data-format.md

@@ -48,6 +48,11 @@ to its URL: `../` for `stories/7.json`, `../../` for `stories/tw/7.json`. Withou
 The loader's `base` option overrides that root. Content-addressed files may use gzip or brotli; their entries record
 both decoded `size` and compressed `stored` length.
 
+Part references may name immutable `products/<sha256>.json` manifests. The part's product and subject are checked
+before loading, and its files remain relative to the site root. A site snapshot's `parts.ui`, `parts.fonts` and
+optional `parts.simpleui` list the same set of available languages. Logical files and object JSON roles may use
+the [split file-entry form](data-format.md#file-entries).
+
 ## stories.json
 
 The optional listing index has `format: "ournotes.stories/1"` and a `stories` array. The player loads the manifest

schema/product.schema.json

@@ -86,6 +86,9 @@
         },
         {
           "$ref": "#/$defs/encoded"
+        },
+        {
+          "$ref": "#/$defs/parts"
         }
       ]
     },
@@ -143,6 +146,62 @@
           "$ref": "#/$defs/stored"
         }
       }
+    },
+    "parts": {
+      "type": "object",
+      "required": [
+        "parts",
+        "size"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "parts": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "anyOf": [
+              {
+                "type": "array",
+                "prefixItems": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "$ref": "#/$defs/asset"
+                  },
+                  {
+                    "$ref": "#/$defs/size"
+                  }
+                ],
+                "minItems": 3,
+                "maxItems": 3
+              },
+              {
+                "type": "array",
+                "prefixItems": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "$ref": "#/$defs/encodedAsset"
+                  },
+                  {
+                    "$ref": "#/$defs/size"
+                  },
+                  {
+                    "$ref": "#/$defs/stored"
+                  }
+                ],
+                "minItems": 4,
+                "maxItems": 4
+              }
+            ]
+          }
+        },
+        "size": {
+          "$ref": "#/$defs/size"
+        }
+      }
     }
   }
 }

schema/story-manifest.schema.json

@@ -85,6 +85,9 @@
         },
         {
           "$ref": "#/$defs/encoded"
+        },
+        {
+          "$ref": "#/$defs/parts"
         }
       ]
     },
@@ -142,6 +145,62 @@
           "$ref": "#/$defs/stored"
         }
       }
+    },
+    "parts": {
+      "type": "object",
+      "required": [
+        "parts",
+        "size"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "parts": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "anyOf": [
+              {
+                "type": "array",
+                "prefixItems": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "$ref": "#/$defs/asset"
+                  },
+                  {
+                    "$ref": "#/$defs/size"
+                  }
+                ],
+                "minItems": 3,
+                "maxItems": 3
+              },
+              {
+                "type": "array",
+                "prefixItems": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "$ref": "#/$defs/encodedAsset"
+                  },
+                  {
+                    "$ref": "#/$defs/size"
+                  },
+                  {
+                    "$ref": "#/$defs/stored"
+                  }
+                ],
+                "minItems": 4,
+                "maxItems": 4
+              }
+            ]
+          }
+        },
+        "size": {
+          "$ref": "#/$defs/size"
+        }
+      }
     }
   }
 }

scripts/validate-data.mjs

@@ -285,7 +285,7 @@ class Chart {
   checkAssets() {
     for (const [p, f] of Object.entries(this.files)) {
       if (f.parts) {
-        if (!p.endsWith(".json")) this.err(p, "only JSON files may be split into parts");
+        if (this.extOfFile(p) !== "json") this.err(p, "only JSON files may be split into parts");
         const keys = new Set();
         let total = 2 + f.parts.length - 1;        // braces and commas
         for (const [k, a, size, stored] of f.parts) {
@@ -743,10 +743,16 @@ class Product extends Chart {
     for (const [oid, o] of Object.entries(this.objects))
       if (isObj(o) && isObj(o.files)) for (const [role, f] of Object.entries(o.files)) this.files[`${oid}#${role}`] = f;
     this.read = new Set();            // the files a reader of the product reads
+    this.immutable = /^products\/[0-9a-f]{64}\.json$/.test(this.where || "");
+    if (this.immutable) {
+      const actual = crypto.createHash("sha256").update(fs.readFileSync(site.file(this.where))).digest("hex");
+      if (path.posix.basename(this.where, ".json") !== actual) this.err("manifest", "content id differs from the immutable manifest path");
+    }
   }
 
   // the manifest's name (its path without the directory and .json) is its subject with "/" as "_"
   checkName() {
+    if (this.immutable) return;
     const name = path.posix.basename(this.where, ".json"), want = String(this.man.subject).replaceAll("/", "_");
     if (name !== want) this.err("manifest", `subject ${this.man.subject}: the manifest's name is ${name}, not ${want}`);
   }
@@ -756,7 +762,7 @@ class Product extends Chart {
     const cut = p.indexOf("#");
     if (cut < 0) return extOf(p);
     const role = p.slice(cut + 1);
-    return role === "json" || role === "index" ? "json" : role === "image" ? "png" : role.startsWith("program:") ? "glsl" : null;
+    return ["json", "index", "arrays"].includes(role) ? "json" : role === "image" ? "png" : role.startsWith("program:") ? "glsl" : null;
   }
 
   // the object `id` is in the manifest, of class `cls` (`where`: the reference's place, for errors)
@@ -815,7 +821,7 @@ class Model extends Product {
     const key = MODEL_KEY.exec(m.subject);
     if (!key) this.err("manifest", `subject ${m.subject} is not a model key (Character/Live2D/<group>/<name>/model/<name>)`);
     const name = this.where ? path.posix.basename(this.where, ".json") : null;
-    if (key && name !== null && key[2] !== name) this.err("manifest", `subject: model ${key[2]}, expected ${name} (the manifest's name)`);
+    if (key && name !== null && !this.immutable && key[2] !== name) this.err("manifest", `subject: model ${key[2]}, expected ${name} (the manifest's name)`);
     if (Object.keys(m.files).join("\n") !== "model.json") this.err("manifest", "files: model.json is the one file of a model");
     this.checkAssets();
     if (this.errs.length) return this.errs;

src/live/store.js

@@ -50,7 +50,7 @@ export const loadLiveStore = async (url, { variants = [], fetch = globalThis.fet
     return AssetStore.fromManifest(href, { ...opts, fetch: serve(man), base, onProgress });
   }
   if (!isProduct(man, LIVE_PRODUCT)) throw new Error(`${href}: not a live manifest (${PRODUCT_MANIFEST} of ${LIVE_PRODUCT})`);
-  const root = base ? new URL(String(base), here()) : new URL("../", href);
+  const root = base ? new URL(String(base), here()) : new URL(man.root || "../", href);
   if (!href.startsWith(root.href)) throw new Error(`${href}: not under the root ${root.href}`);
   const liveDir = productDir(href.slice(root.href.length));
   // live.json names the parts: read it first (through fromManifest, which decodes and checks it)

tests/unit/live-store.test.mjs

@@ -66,6 +66,15 @@ const live = () => {
   return L;
 };
 
+test("a regional live manifest resolves products against its declared root", async () => {
+  const L = live(), man = L.files.get("lives/7_expert.json");
+  L.files.set("lives/region/7_expert.json", { ...man, root: "../../" });
+  const store = await loadLiveStore(`${ROOT}lives/region/7_expert.json`, { fetch: L.fetch });
+  assert.equal(store.info.dir, "lives/region/7_expert");
+  assert.deepEqual(new LiveData(store).score().notes, [1]);
+  assert.deepEqual(new LiveData(store).audio().sounds[10].layers[0].file, "sounds/default/audio/se1/a.flac");
+});
+
 test("productDir: a product manifest path without .json", () => {
   assert.equal(productDir("scenes/default.json"), "scenes/default");
   assert.equal(productDir("sounds/NoteSePatternId=1,3.json"), "sounds/NoteSePatternId=1,3");

tests/unit/ui.test.mjs

@@ -140,3 +140,22 @@ test('UI validation checks the product graph, binding names and library referenc
   const bindings=validateUI(n=>edit(n,d=>Object.values(Object.values(Object.values(d.bindings)[0])[0])[0].pop()));assert.equal(bindings.status,1);assert.match(bindings.text,/names, the clip has/);
   const missing=validateUI(n=>{n.index.controllers=[{id:'ui:99999',name:'Missing',manifest:'uipacks/one.json'}];n.files.set('index.json',new TextEncoder().encode(JSON.stringify(n.index)));});assert.equal(missing.status,1);assert.match(missing.text,/not in the store/);
 });
+
+const splitUI=n=>{
+  for(const entry of [...Object.values(n.manifest.files),...Object.values(n.manifest.objects).flatMap(o=>Object.values(o.files))]){
+    if(!entry.asset.endsWith('.json'))continue;
+    const doc=JSON.parse(new TextDecoder().decode(n.files.get(entry.asset))),text=JSON.stringify(doc);
+    const parts=Object.entries(doc).map(([key,value])=>{const p=n.asset(JSON.stringify(value));return [key,p.asset,p.size];});
+    delete entry.asset;entry.parts=parts;entry.size=new TextEncoder().encode(text).length;
+  }
+  n.files.set('uipacks/one.json',new TextEncoder().encode(JSON.stringify(n.manifest)));
+};
+test('product files and object JSON roles reconstruct split entries and validate their sizes',async()=>{
+  const n=network();splitUI(n);
+  const {pack:p}=await new UILibrary(n.index,'https://example.invalid/data/',n.fetch).loadPack('one');
+  assert.equal(p.nodes[0].name,'Root');assert.equal(p.controller(0).clip(0).name,'A');
+  const valid=validateUI(splitUI);assert.equal(valid.status,0,valid.text);
+  const bad=validateUI(n=>{splitUI(n);Object.values(n.manifest.objects)[0].files.json.size++;
+    n.files.set('uipacks/one.json',new TextEncoder().encode(JSON.stringify(n.manifest)));});
+  assert.equal(bad.status,1);assert.match(bad.text,/rebuilt text/);
+});

tests/unit/validate-live.test.mjs

@@ -11,7 +11,7 @@ import { fileURLToPath } from "node:url";
 import { AssetStore } from "../../src/data/assets.js";
 import { legacyLive } from "../../src/live/legacy.js";
 import { AUDIO, NOTES, SCENE, chartFiles, node, sprite } from "./live-legacy-fixture.mjs";
-import { glsl, png, putAsset, putFiles } from "./site-fixture.mjs";
+import { glsl, png, putAsset, putFiles, sha256 } from "./site-fixture.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const script = path.join(here, "..", "..", "scripts", "validate-data.mjs");
@@ -174,6 +174,25 @@ test("a part manifest named other than its subject is reported", () => {
   hasLine(r, "the manifest's name is other, not skin001");
 });
 
+test("immutable part manifests are checked by content identity", () => {
+  const build = damaged => dir => {
+    const docs = productDocs(dir, chart()), part = docs["noteskins/skin001.json"];
+    const target = `products/${sha256(JSON.stringify(part))}.json`;
+    delete docs["noteskins/skin001.json"];
+    docs[target] = damaged ? { ...part, root: "../" } : part;
+    const root = docs["lives/7_expert.json"];
+    const live = JSON.parse(fs.readFileSync(path.join(dir, root.files["live.json"].asset), "utf8"));
+    live.parts.noteskin = target;
+    root.files["live.json"] = putAsset(dir, "live.json", live);
+    return docs;
+  };
+  const valid = run(build(false));
+  assert.equal(valid.status, 0, valid.lines.join("\n"));
+  const invalid = run(build(true));
+  assert.equal(invalid.status, 1);
+  hasLine(invalid, "content id differs from the immutable manifest path");
+});
+
 test("two lives sharing one broken part report its errors once", () => {
   const r = run((dir) => {
     const docs = productDocs(dir, chart(), { edit: { "noteskins/skin001/noteskin.json": "{" } });

tests/unit/validate-manifest.test.mjs

@@ -96,7 +96,7 @@ test("chart manifests: formats 2 and 3, file entries stored as they are or encod
   assert.ok(schema("manifest")(manifest({ files: { "live.json": { asset: `assets/${a}.json.gz`, size: 100 } } })).length > 0);
 });
 
-test("product manifests and model.json: object tables of whole or encoded files; model.json format 3", () => {
+test("product manifests and model.json: whole, encoded and split files; model.json format 3", () => {
   const a = "a".repeat(64), S = schema("product");
   const man = { schema: "nnnotes.product/1", product: "live2d.model", subject: KEY,
                 files: { "model.json": { asset: `assets/${a}.json.gz`, size: 100, stored: 60 } },
@@ -105,7 +105,9 @@ test("product manifests and model.json: object tables of whole or encoded files;
   assert.deepEqual(S(man), []);
   assert.ok(S({ ...man, schema: "nnnotes.product/2" }).some((e) => e.includes("/schema")));
   assert.ok(S({ ...man, objects: { "CAB-x": man.objects["CAB-x:-12"] } }).length > 0);              // not an object id
-  assert.ok(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`, 5]], size: 7 } } }).length > 0);   // no parts
+  assert.deepEqual(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`, 5]], size: 11 } } }), []);
+  assert.deepEqual(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json.gz`, 5, 3]], size: 11 } } }), []);
+  assert.ok(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`]], size: 11 } } }).length > 0);
   const { model } = modelObjects("m1");
   assert.deepEqual(schema("model-json")(model), []);
   assert.ok(schema("model-json")({ ...model, format: 2 }).some((e) => e.includes("/format")));