diff --git a/docs/data-format.md b/docs/data-format.md index 55861fd..15e53b6 100644 --- a/docs/data-format.md +++ b/docs/data-format.md @@ -90,7 +90,7 @@ A file entry has one of these forms: `.br`). `size` is the byte count of the file, `stored` that of the asset as stored (the encoded bytes). An asset is encoded only when that makes it smaller, so `stored` is less than `size`; `stored` is present exactly when the asset name ends in `.gz` or `.br`. -- **Split JSON object** (a chart manifest's files only): `{ "parts": [[key, asset, size], ...], "size": }`; a +- **Split JSON object** (logical files and object JSON roles): `{ "parts": [[key, asset, size], ...], "size": }`; a part whose asset is encoded is `[key, asset, size, stored]`. A large JSON file whose top level is an object is stored per top-level key: each part's asset holds the raw JSON text of that key's value. The file's text is rebuilt as @@ -123,6 +123,17 @@ Several products may retain different file roles of the same object. Loading the roles; overlapping roles and object metadata must agree. Product paths may include region directories, and are relative to the layout root. +### Immutable product manifests + +A site can store a product manifest at `products/.json`, with the SHA-256 of the manifest bytes as its +name. Its parent product's `parts` fields name that exact path, so each public entry point selects a coherent +set of dependencies. Public aliases may remain at `lives/.json`, `models/.json` or `stories/.json`. +An alias in a region directory sets `root: "../../"`; a two-level path uses `"../"`. The live and story loaders +honor that root, and a supplied `base` overrides it. + +The data validator checks immutable manifest contents against their names, then applies the product, subject, +object and file checks. Ordinary product-layout manifest names continue to identify their subjects. + An object is one Unity object, converted. Its id is `:` (the path id as decimal digits, possibly negative), the same in every product that uses it. An object's JSON names another object by a reference `{ "$ref": { "file": "", "pathId": "" } }`, the object `:`; `null` names none. diff --git a/docs/story-data-format.md b/docs/story-data-format.md index a81e80c..c84973f 100644 --- a/docs/story-data-format.md +++ b/docs/story-data-format.md @@ -48,6 +48,11 @@ to its URL: `../` for `stories/7.json`, `../../` for `stories/tw/7.json`. Withou The loader's `base` option overrides that root. Content-addressed files may use gzip or brotli; their entries record both decoded `size` and compressed `stored` length. +Part references may name immutable `products/.json` manifests. The part's product and subject are checked +before loading, and its files remain relative to the site root. A site snapshot's `parts.ui`, `parts.fonts` and +optional `parts.simpleui` list the same set of available languages. Logical files and object JSON roles may use +the [split file-entry form](data-format.md#file-entries). + ## stories.json The optional listing index has `format: "ournotes.stories/1"` and a `stories` array. The player loads the manifest diff --git a/schema/product.schema.json b/schema/product.schema.json index a2da330..1657e58 100644 --- a/schema/product.schema.json +++ b/schema/product.schema.json @@ -86,6 +86,9 @@ }, { "$ref": "#/$defs/encoded" + }, + { + "$ref": "#/$defs/parts" } ] }, @@ -143,6 +146,62 @@ "$ref": "#/$defs/stored" } } + }, + "parts": { + "type": "object", + "required": [ + "parts", + "size" + ], + "additionalProperties": false, + "properties": { + "parts": { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/asset" + }, + { + "$ref": "#/$defs/size" + } + ], + "minItems": 3, + "maxItems": 3 + }, + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/encodedAsset" + }, + { + "$ref": "#/$defs/size" + }, + { + "$ref": "#/$defs/stored" + } + ], + "minItems": 4, + "maxItems": 4 + } + ] + } + }, + "size": { + "$ref": "#/$defs/size" + } + } } } } diff --git a/schema/story-manifest.schema.json b/schema/story-manifest.schema.json index aa77103..db4bbf0 100644 --- a/schema/story-manifest.schema.json +++ b/schema/story-manifest.schema.json @@ -85,6 +85,9 @@ }, { "$ref": "#/$defs/encoded" + }, + { + "$ref": "#/$defs/parts" } ] }, @@ -142,6 +145,62 @@ "$ref": "#/$defs/stored" } } + }, + "parts": { + "type": "object", + "required": [ + "parts", + "size" + ], + "additionalProperties": false, + "properties": { + "parts": { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/asset" + }, + { + "$ref": "#/$defs/size" + } + ], + "minItems": 3, + "maxItems": 3 + }, + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/encodedAsset" + }, + { + "$ref": "#/$defs/size" + }, + { + "$ref": "#/$defs/stored" + } + ], + "minItems": 4, + "maxItems": 4 + } + ] + } + }, + "size": { + "$ref": "#/$defs/size" + } + } } } } diff --git a/scripts/validate-data.mjs b/scripts/validate-data.mjs index 2cc64c5..7ed1111 100644 --- a/scripts/validate-data.mjs +++ b/scripts/validate-data.mjs @@ -285,7 +285,7 @@ class Chart { checkAssets() { for (const [p, f] of Object.entries(this.files)) { if (f.parts) { - if (!p.endsWith(".json")) this.err(p, "only JSON files may be split into parts"); + if (this.extOfFile(p) !== "json") this.err(p, "only JSON files may be split into parts"); const keys = new Set(); let total = 2 + f.parts.length - 1; // braces and commas for (const [k, a, size, stored] of f.parts) { @@ -743,10 +743,16 @@ class Product extends Chart { for (const [oid, o] of Object.entries(this.objects)) if (isObj(o) && isObj(o.files)) for (const [role, f] of Object.entries(o.files)) this.files[`${oid}#${role}`] = f; this.read = new Set(); // the files a reader of the product reads + this.immutable = /^products\/[0-9a-f]{64}\.json$/.test(this.where || ""); + if (this.immutable) { + const actual = crypto.createHash("sha256").update(fs.readFileSync(site.file(this.where))).digest("hex"); + if (path.posix.basename(this.where, ".json") !== actual) this.err("manifest", "content id differs from the immutable manifest path"); + } } // the manifest's name (its path without the directory and .json) is its subject with "/" as "_" checkName() { + if (this.immutable) return; const name = path.posix.basename(this.where, ".json"), want = String(this.man.subject).replaceAll("/", "_"); if (name !== want) this.err("manifest", `subject ${this.man.subject}: the manifest's name is ${name}, not ${want}`); } @@ -756,7 +762,7 @@ class Product extends Chart { const cut = p.indexOf("#"); if (cut < 0) return extOf(p); const role = p.slice(cut + 1); - return role === "json" || role === "index" ? "json" : role === "image" ? "png" : role.startsWith("program:") ? "glsl" : null; + return ["json", "index", "arrays"].includes(role) ? "json" : role === "image" ? "png" : role.startsWith("program:") ? "glsl" : null; } // the object `id` is in the manifest, of class `cls` (`where`: the reference's place, for errors) @@ -815,7 +821,7 @@ class Model extends Product { const key = MODEL_KEY.exec(m.subject); if (!key) this.err("manifest", `subject ${m.subject} is not a model key (Character/Live2D///model/)`); const name = this.where ? path.posix.basename(this.where, ".json") : null; - if (key && name !== null && key[2] !== name) this.err("manifest", `subject: model ${key[2]}, expected ${name} (the manifest's name)`); + if (key && name !== null && !this.immutable && key[2] !== name) this.err("manifest", `subject: model ${key[2]}, expected ${name} (the manifest's name)`); if (Object.keys(m.files).join("\n") !== "model.json") this.err("manifest", "files: model.json is the one file of a model"); this.checkAssets(); if (this.errs.length) return this.errs; diff --git a/src/live/store.js b/src/live/store.js index 7d10efd..3f8da0f 100644 --- a/src/live/store.js +++ b/src/live/store.js @@ -50,7 +50,7 @@ export const loadLiveStore = async (url, { variants = [], fetch = globalThis.fet return AssetStore.fromManifest(href, { ...opts, fetch: serve(man), base, onProgress }); } if (!isProduct(man, LIVE_PRODUCT)) throw new Error(`${href}: not a live manifest (${PRODUCT_MANIFEST} of ${LIVE_PRODUCT})`); - const root = base ? new URL(String(base), here()) : new URL("../", href); + const root = base ? new URL(String(base), here()) : new URL(man.root || "../", href); if (!href.startsWith(root.href)) throw new Error(`${href}: not under the root ${root.href}`); const liveDir = productDir(href.slice(root.href.length)); // live.json names the parts: read it first (through fromManifest, which decodes and checks it) diff --git a/tests/unit/live-store.test.mjs b/tests/unit/live-store.test.mjs index 2239303..afe9fd1 100644 --- a/tests/unit/live-store.test.mjs +++ b/tests/unit/live-store.test.mjs @@ -66,6 +66,15 @@ const live = () => { return L; }; +test("a regional live manifest resolves products against its declared root", async () => { + const L = live(), man = L.files.get("lives/7_expert.json"); + L.files.set("lives/region/7_expert.json", { ...man, root: "../../" }); + const store = await loadLiveStore(`${ROOT}lives/region/7_expert.json`, { fetch: L.fetch }); + assert.equal(store.info.dir, "lives/region/7_expert"); + assert.deepEqual(new LiveData(store).score().notes, [1]); + assert.deepEqual(new LiveData(store).audio().sounds[10].layers[0].file, "sounds/default/audio/se1/a.flac"); +}); + test("productDir: a product manifest path without .json", () => { assert.equal(productDir("scenes/default.json"), "scenes/default"); assert.equal(productDir("sounds/NoteSePatternId=1,3.json"), "sounds/NoteSePatternId=1,3"); diff --git a/tests/unit/ui.test.mjs b/tests/unit/ui.test.mjs index 4dd4540..d9b72d6 100644 --- a/tests/unit/ui.test.mjs +++ b/tests/unit/ui.test.mjs @@ -140,3 +140,22 @@ test('UI validation checks the product graph, binding names and library referenc const bindings=validateUI(n=>edit(n,d=>Object.values(Object.values(Object.values(d.bindings)[0])[0])[0].pop()));assert.equal(bindings.status,1);assert.match(bindings.text,/names, the clip has/); const missing=validateUI(n=>{n.index.controllers=[{id:'ui:99999',name:'Missing',manifest:'uipacks/one.json'}];n.files.set('index.json',new TextEncoder().encode(JSON.stringify(n.index)));});assert.equal(missing.status,1);assert.match(missing.text,/not in the store/); }); + +const splitUI=n=>{ + for(const entry of [...Object.values(n.manifest.files),...Object.values(n.manifest.objects).flatMap(o=>Object.values(o.files))]){ + if(!entry.asset.endsWith('.json'))continue; + const doc=JSON.parse(new TextDecoder().decode(n.files.get(entry.asset))),text=JSON.stringify(doc); + const parts=Object.entries(doc).map(([key,value])=>{const p=n.asset(JSON.stringify(value));return [key,p.asset,p.size];}); + delete entry.asset;entry.parts=parts;entry.size=new TextEncoder().encode(text).length; + } + n.files.set('uipacks/one.json',new TextEncoder().encode(JSON.stringify(n.manifest))); +}; +test('product files and object JSON roles reconstruct split entries and validate their sizes',async()=>{ + const n=network();splitUI(n); + const {pack:p}=await new UILibrary(n.index,'https://example.invalid/data/',n.fetch).loadPack('one'); + assert.equal(p.nodes[0].name,'Root');assert.equal(p.controller(0).clip(0).name,'A'); + const valid=validateUI(splitUI);assert.equal(valid.status,0,valid.text); + const bad=validateUI(n=>{splitUI(n);Object.values(n.manifest.objects)[0].files.json.size++; + n.files.set('uipacks/one.json',new TextEncoder().encode(JSON.stringify(n.manifest)));}); + assert.equal(bad.status,1);assert.match(bad.text,/rebuilt text/); +}); diff --git a/tests/unit/validate-live.test.mjs b/tests/unit/validate-live.test.mjs index ad7c362..bb47a03 100644 --- a/tests/unit/validate-live.test.mjs +++ b/tests/unit/validate-live.test.mjs @@ -11,7 +11,7 @@ import { fileURLToPath } from "node:url"; import { AssetStore } from "../../src/data/assets.js"; import { legacyLive } from "../../src/live/legacy.js"; import { AUDIO, NOTES, SCENE, chartFiles, node, sprite } from "./live-legacy-fixture.mjs"; -import { glsl, png, putAsset, putFiles } from "./site-fixture.mjs"; +import { glsl, png, putAsset, putFiles, sha256 } from "./site-fixture.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); const script = path.join(here, "..", "..", "scripts", "validate-data.mjs"); @@ -174,6 +174,25 @@ test("a part manifest named other than its subject is reported", () => { hasLine(r, "the manifest's name is other, not skin001"); }); +test("immutable part manifests are checked by content identity", () => { + const build = damaged => dir => { + const docs = productDocs(dir, chart()), part = docs["noteskins/skin001.json"]; + const target = `products/${sha256(JSON.stringify(part))}.json`; + delete docs["noteskins/skin001.json"]; + docs[target] = damaged ? { ...part, root: "../" } : part; + const root = docs["lives/7_expert.json"]; + const live = JSON.parse(fs.readFileSync(path.join(dir, root.files["live.json"].asset), "utf8")); + live.parts.noteskin = target; + root.files["live.json"] = putAsset(dir, "live.json", live); + return docs; + }; + const valid = run(build(false)); + assert.equal(valid.status, 0, valid.lines.join("\n")); + const invalid = run(build(true)); + assert.equal(invalid.status, 1); + hasLine(invalid, "content id differs from the immutable manifest path"); +}); + test("two lives sharing one broken part report its errors once", () => { const r = run((dir) => { const docs = productDocs(dir, chart(), { edit: { "noteskins/skin001/noteskin.json": "{" } }); diff --git a/tests/unit/validate-manifest.test.mjs b/tests/unit/validate-manifest.test.mjs index 1600d55..d3d04a6 100644 --- a/tests/unit/validate-manifest.test.mjs +++ b/tests/unit/validate-manifest.test.mjs @@ -96,7 +96,7 @@ test("chart manifests: formats 2 and 3, file entries stored as they are or encod assert.ok(schema("manifest")(manifest({ files: { "live.json": { asset: `assets/${a}.json.gz`, size: 100 } } })).length > 0); }); -test("product manifests and model.json: object tables of whole or encoded files; model.json format 3", () => { +test("product manifests and model.json: whole, encoded and split files; model.json format 3", () => { const a = "a".repeat(64), S = schema("product"); const man = { schema: "nnnotes.product/1", product: "live2d.model", subject: KEY, files: { "model.json": { asset: `assets/${a}.json.gz`, size: 100, stored: 60 } }, @@ -105,7 +105,9 @@ test("product manifests and model.json: object tables of whole or encoded files; assert.deepEqual(S(man), []); assert.ok(S({ ...man, schema: "nnnotes.product/2" }).some((e) => e.includes("/schema"))); assert.ok(S({ ...man, objects: { "CAB-x": man.objects["CAB-x:-12"] } }).length > 0); // not an object id - assert.ok(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`, 5]], size: 7 } } }).length > 0); // no parts + assert.deepEqual(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`, 5]], size: 11 } } }), []); + assert.deepEqual(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json.gz`, 5, 3]], size: 11 } } }), []); + assert.ok(S({ ...man, files: { "n.json": { parts: [["k", `assets/${a}.json`]], size: 11 } } }).length > 0); const { model } = modelObjects("m1"); assert.deepEqual(schema("model-json")(model), []); assert.ok(schema("model-json")({ ...model, format: 2 }).some((e) => e.includes("/format")));