From edae9332b27d5973584f594d11bf37c23ece1e8d Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:28:08 +0800 Subject: [PATCH 01/15] feat: add reproducible conversation extraction and independent text publications --- README.md | 7 + docs/pipeline.md | 127 ++ docs/text-localization.md | 60 + pyproject.toml | 2 +- src/chara/common_talks.py | 114 ++ src/chara/communication_actions.py | 107 ++ src/chara/talks.py | 11 +- src/chara/text_catalog.py | 260 ++++ src/cli/__init__.py | 1 + src/cli/__main__.py | 3 + src/cli/legacy.py | 1721 +++++++++++++++++++++++++++ src/cli/main.py | 18 + src/cli/pipeline.py | 131 ++ src/core/assets/router.py | 2 + src/core/cli.py | 1687 +------------------------- src/core/content.py | 20 + src/core/extract.py | 7 + src/pack/gc.py | 15 + src/pack/groups.py | 4 +- src/pack/identity.py | 7 +- src/pack/task_results.py | 56 + src/pack/verify.py | 10 + src/perf/fixture_talks.py | 2 +- src/phenomena/audio_corpus.py | 1 + src/pipeline/__init__.py | 1 + src/pipeline/contract.py | 44 + src/pipeline/depot.py | 86 ++ src/pipeline/stages.py | 171 +++ src/pipeline/text_product.py | 95 ++ src/source/__init__.py | 1 + src/source/snapshot.py | 47 + tests/check_distribution.py | 2 +- tests/test_birthday_party_talks.py | 12 +- tests/test_common_talks.py | 75 ++ tests/test_motion_storage.py | 3 +- tests/test_pipeline.py | 153 +++ tests/test_text_catalog.py | 84 ++ tools/deployment/upload_oss.py | 2 +- tools/migrate_canonical_metadata.py | 5 +- 39 files changed, 3449 insertions(+), 1705 deletions(-) create mode 100644 docs/pipeline.md create mode 100644 docs/text-localization.md create mode 100644 src/chara/common_talks.py create mode 100644 src/chara/communication_actions.py create mode 100644 src/chara/text_catalog.py create mode 100644 src/cli/__init__.py create mode 100644 src/cli/__main__.py create mode 100644 src/cli/legacy.py create mode 100644 src/cli/main.py create mode 100644 src/cli/pipeline.py create mode 100644 src/core/content.py create mode 100644 src/pack/task_results.py create mode 100644 src/pipeline/__init__.py create mode 100644 src/pipeline/contract.py create mode 100644 src/pipeline/depot.py create mode 100644 src/pipeline/stages.py create mode 100644 src/pipeline/text_product.py create mode 100644 src/source/__init__.py create mode 100644 src/source/snapshot.py create mode 100644 tests/test_common_talks.py create mode 100644 tests/test_pipeline.py create mode 100644 tests/test_text_catalog.py diff --git a/README.md b/README.md index 2ea8bb3..ab86cf8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # moly-root +## 可复用的提取任务与文本产品 + +`moly source` 记录输入快照；`moly export talks|text-catalog|official-text|text-runtime` +通过有版本的任务提取对话、维护文本身份、匹配其他区服官方文本并输出按脚本加载的显示分片。 +`moly depot plan|run|verify|layout|gc` 管理任务复用与校验，复用现有 pack 存储。 +旧的平面命令继续可用。使用方式、缓存边界与源目录保留要求见 [提取管线](docs/pipeline.md)。 + ## 玩家布局导入资源 从对应区服的主表和已解密家具包生成导入目录及配色贴图： diff --git a/docs/pipeline.md b/docs/pipeline.md new file mode 100644 index 0000000..02fc97e --- /dev/null +++ b/docs/pipeline.md @@ -0,0 +1,127 @@ +# Reproducible extraction tasks + +The extraction pipeline follows the same separation as +[nnnotes](https://github.com/MetaSekaiLab/nnnotes): versioned sources, reusable +tasks, immutable results, and product layouts. The existing Moly pack v2 +contracts and logical runtime paths remain the publication boundary. + +`cli` parses options and calls library code. `source` records and checks input +bytes. `pipeline` composes the existing domain readers, text identity registry, +official-text matcher and display products. `pack` owns the shared blob store +and release retention. Legacy domain exports remain available; they have not +all been converted into cached tasks. + +## Sources and plans + +```sh +moly source record --region jp --version VERSION \ + --input talks=inputs/talk.bundle --input constants=inputs/lib.bundle \ + --out source.json +moly source verify source.json + +moly export talks --master inputs/master --bundle inputs/talk.bundle \ + --lib inputs/lib.bundle --depot asset-store --out extracted \ + --plan --emit-task talks-task.json +moly depot run --depot asset-store --task talks-task.json +``` + +The source receipt and task descriptions contain local input locators. Keep +these in build storage, outside runtime publication. Copying them to another +machine requires replacing their locators with paths to the same bytes; this +does not change the task key. Source receipts identify a caller-supplied region +and version and do not authenticate a publisher or discover new game files. + +`--family fixture` selects the existing furniture dialogue reader. The default +includes solo, birthday and ordinary multi-character conversations. Both use +the existing script and constant readers. The Unity fallback version is an +explicit parameter, not evidence of the actual bundle's engine version. + +`moly export communication-actions --master inputs/master --bundle inputs/communication.bundle +--depot asset-store --out extracted` exports the separate per-character +communication motion loops. It retains participant conditions and random +branches; these scripts contain presentation actions rather than spoken text. +The manifest-driven extractor also routes this family automatically. + +A task key hashes its stage/version, subject, normalized parameters, actual +input hashes and sizes, used implementation/dependency identifiers, and +consumed context. Locators and output directories do not affect it. Source +module hashes distinguish unreleased edits as well as installed versions. +`depot plan --previous old-task.json` explains which key components changed. + +`depot run` rejects incompatible installed stages, changed input bytes, corrupt +cached outputs and forged task keys. It verifies and freezes inputs before +calling readers. `--force` recomputes a result and refuses different output for +the same key. Failures leave no committed result. This first implementation +runs one task at a time under the store's existing publication lock. + +## Text products + +```sh +moly export text-catalog --corpus extracted/talks.json \ + --corpus extracted/fixture-talks/talks.json --region jp \ + --source-revision SOURCE_REVISION --previous prior/text-source.json \ + --depot asset-store --out text-jp +moly export official-text --source text-jp/text-source.json \ + --donor text-cn/text-source.json --locale zh-CN \ + --depot asset-store --out official +moly export text-runtime --source text-jp/text-source.json \ + --overlay official/official-zh-CN.json --depot asset-store --out display +``` + +Omit `--previous` only for the first catalogue. The previous source catalogue +is the durable identity registry used for future updates; preserve it and its +retired history. Text identities and source-change handling are described in +[text localization](text-localization.md). + +`text/index.json` is a `moly-text-runtime/1` root. Hashing a script key as UTF-8 +SHA-256 and taking the first two hex digits selects one of at most 256 routing +buckets. Each `moly-text-routes/1` bucket maps script keys to a display source. +Every reference carries `path`, `sha256` and +`bytes`. Objects live at `text/objects/.json`; discovery is written last. + +Display sources carry only original text, identities, source hashes and current +step bindings. A script's linked tweets are in the same display source. The +complete registry, editor context and retired history are not downloaded for +each playback. Shards have a 2 MiB bound. Unchanged scripts retain identical +bytes when another source script or translation changes. The source root never +contains a translation revision. Publish only `text/` with the resource snapshot. + +Official overlays are a separate tree at `translations///`. +Its `latest.json` (`moly-text-publication/1`) and archived +`revisions/.json` use the same 256-bucket routing. Translation buckets +(`moly-text-translation-routes/1`) map script keys directly to overlay references; +all references are relative to that locale directory. Publish this tree +independently. Editing translations changes neither the source index nor the +resource snapshot. Sessions pin a publication, immutable objects are reusable, +and source ID/hash compatibility is checked per line, not by requiring the +whole source registry revision to match. Retain historical publication roots +and their objects for playback pins and rollback. + +These exports prepare inputs for the editorial and playback systems. They do +not publish translations or grant approval to drafts. + +## One store, two kinds of roots + +```text +asset-store/ + blobs/ab/.bin extraction bytes (existing identity codec) + tasks/.json committed moly-result/1, with verified artifacts + packages/ existing release packages + catalogs/ existing retained releases + channels/ existing release selectors +``` + +Task results contain key parts and artifact records, never local input paths +or timings. Both extraction results and retained release catalogs protect +their referenced blobs. `moly depot verify --depot asset-store` verifies all +task results; `python -m pack.verify --out asset-store` verifies release roots +and task roots together when a release exists. `moly depot gc` uses the same +collector as `pack.gc`; it reports only unless `--apply` is passed. It refuses +collection on corrupt roots or an incomplete release layout. Task history is +retained; this interface does not implicitly retire prior source registries. + +`depot layout --key KEY --out DIRECTORY` copies verified artifacts into an +editable output directory, separate from the store. It never hard-links that +directory to shared immutable blobs. Use the existing pack/group publication +workflow for runtime releases. Content/representation identity, JSON number +semantics, existing domain schemas and release channels are unchanged. diff --git a/docs/text-localization.md b/docs/text-localization.md new file mode 100644 index 0000000..433d1af --- /dev/null +++ b/docs/text-localization.md @@ -0,0 +1,60 @@ +# Display text and ordinary multi-character conversations + +`moly talks` also exports ordinary conversations between two to five characters +under `commonSomeCharacter`. These talks have no fixture anchor. They retain +participant slots, source association rows, conditions, pre-actions, tweets, +script steps and voice cues. The runtime chooses the initiating character; +association metadata does not fix that conversation's leader. + +The audio corpus reader includes this block. Existing solo `units` and +`birthdayParty` blocks keep their own semantics. A consumer must explicitly +support the new family before it can play these conversations. + +## Text catalogues + +Generate a translation source from caller-provided extracted documents: + +```sh +moly text-catalog --corpus extracted/talks.json \ + --corpus extracted/fixture-talks/talks.json \ + --region jp --source-revision SOURCE_REVISION --out text-source.json +``` + +On the next source update pass `--previous text-source-previous.json`. Keep that +previous catalogue in durable versioned storage: it is the registry that +preserves text identities across updates. The output identifies scripts and +display units, with original text, source hashes, speakers, voice context and +bindings to the current script steps. Step numbers are bindings, not identity. + +Unique voice anchors preserve identity when the text changes; unchanged text +can also retain an unambiguous identity. Changes invalidate the old source hash. +Ambiguous repeated lines after a source edit are retired and allocated new IDs +for review. A display-only label does not replace the original control label. +Tweets have their own master identities because multiple talk records can share +a script but reference different tweets. + +The catalogue may contain original game text and derived context. It belongs in +the caller's asset/content storage, not in this source repository. + +## Official translations + +Generate a regional official overlay from two catalogues: + +```sh +moly official-text --source jp-text-source.json --donor cn-text-source.json \ + --locale zh-CN --out official-zh-CN.json +``` + +The matcher requires compatible script identity, participant sets and per-line +speaker/voice context. A same-numbered talk is not enough. Unanchored or +incompatible lines are listed in `issues` for explicit review. A tweet also +needs matching source metadata and script associations. Identical or empty +donor text is not silently labelled a translation. + +The overlay contains plain text, source IDs/hashes and donor provenance. It +never changes executable steps, voice cues, timing or region assets. A text +consumer can prefer matching official text, then a published community +translation, then the original. Source region and display locale are separate. + +These commands prepare source catalogues and official candidates. They do not +publish a service, grant editorial approval, or verify runtime playback. diff --git a/pyproject.toml b/pyproject.toml index 3cee376..bc5e959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ jp = [ ] [project.scripts] -moly = "core.cli:main" +moly = "cli.main:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/chara/common_talks.py b/src/chara/common_talks.py new file mode 100644 index 0000000..4cb7a7e --- /dev/null +++ b/src/chara/common_talks.py @@ -0,0 +1,114 @@ +"""Ordinary conversations between multiple characters, without a fixture. + +Membership follows the source's IsCommonSomeCharacterTalk predicate: there +are two to five nonzero group members, no fixture-id condition, and a row in +the some-character association table. Association rows are preserved as +metadata; the runtime's initiating character leads the resulting plan. +Scripts share the ordinary talk parser and voice-bank rule. +""" +from collections import Counter, defaultdict + + +def select_common_talks(master): + groups = {} + for row in master.table("mysekaiGameCharacterUnitGroups"): + key = row["id"] + if key in groups: + raise ValueError(f"duplicate character group {key}") + groups[key] = [row[f"gameCharacterUnitId{i}"] for i in range(1, 6) + if row.get(f"gameCharacterUnitId{i}")] + conditions = master.condition_types() + candidates = [] + for talk in master.table("mysekaiCharacterTalks"): + members = groups.get(talk["mysekaiGameCharacterUnitGroupId"]) + if members is None: + raise ValueError(f"talk {talk['id']}: missing character group") + if not 2 <= len(members) <= 5: + continue + kinds = conditions.get(talk["mysekaiCharacterTalkConditionGroupId"]) + if kinds is None or any(not isinstance(kind, str) for kind in kinds): + raise ValueError(f"talk {talk['id']}: unresolved conditions") + # IsFixtureActionTalk tests this exact condition, not a string prefix + # and not the wider furniture predicate used by the solo selector. + if "mysekai_fixture_id" not in kinds: + candidates.append((talk, members)) + if not candidates: + return [], {"candidates": 0, "selected": 0, "withoutAssociation": []} + associations = defaultdict(list) + for row in master.table("mysekaiCharacterTalkSomeCharacterTalks"): + associations[row["mysekaiCharacterTalkId"]].append(row) + selected, absent = [], [] + for talk, members in candidates: + links = associations.get(talk["id"]) + if not links: + absent.append(talk["id"]) + continue + selected.append({"talk": talk, "unitIds": members, "associations": links}) + return selected, {"candidates": len(candidates), "selected": len(selected), + "withoutAssociation": absent} + + +def common_talks(master, assets, tables, scalars): + """Build the commonSomeCharacter block of the ordinary talk document.""" + from chara.talks import (_asset_named, _tweet, constant_census, parse_script, + TALK_VOICE_BANK) + from perf.fixture_talks import classify_ops + + selected, selection = select_common_talks(master) + rows, scripts, operations = [], [], Counter() + conditions = master.condition_types() + payloads = master.condition_entries() + tweets = master.tweets() + pre_actions = {} + for row in master.table("mysekaiCharacterTalkPreActions"): + pre_actions.setdefault(row["mysekaiCharacterTalkId"], row) + for item in selected: + source = item["talk"] + lua = source["lua"] + name = lua if lua.endswith(".lua") else lua + ".lua" + script = _asset_named(assets, name) + if script is None: + raise LookupError(f"talk {source['id']}: script asset not found: {name}") + _, unknown = classify_ops(script, name) + if unknown: + raise ValueError(f"talk {source['id']}: unknown script operations: {dict(unknown)}") + parsed = parse_script(script, tables, scalars, script_name=name) + pre_action = pre_actions.get(source["id"]) + tweet_id = (pre_action or {}).get("mysekaiCharacterTalkTweetId") + if tweet_id and tweet_id not in tweets: + raise LookupError(f"talk {source['id']}: tweet not found: {tweet_id}") + group = source["mysekaiCharacterTalkConditionGroupId"] + rows.append({ + "talkId": source["id"], "lua": lua, + "unitGroupId": source["mysekaiGameCharacterUnitGroupId"], + "unitIds": item["unitIds"], + "someCharacterTalks": item["associations"], + "siteGroupId": source["mysekaiSiteGroupId"], + "termId": source["mysekaiCharacterTalkTermId"], + "conditionGroupId": group, "conditions": conditions[group], + "conditionValues": payloads[group], + "preAction": pre_action, "tweet": _tweet(tweets.get(tweet_id)), + "voiceBank": TALK_VOICE_BANK.format(lua.removesuffix(".lua")), + "voices": parsed["voices"], "steps": parsed["steps"], + }) + scripts.append(script) + operations.update(parsed["operations"]) + constants = constant_census(scripts, tables) + summary = { + "talks": len(rows), "steps": sum(len(row["steps"]) for row in rows), + "voiceCues": sum(len(row["voices"]) for row in rows), + "uniqueVoiceCues": len({cue for row in rows for cue in row["voices"]}), + "operations": dict(operations), "selection": selection, + "constants": constants, + } + return { + "version": 1, + "semantics": { + "membership": "Non-fixture talks with two to five group members and a some-character association.", + "mainCharacter": "Chosen by the runtime: the initiating character leads the plan; association metadata does not fix its leader.", + "unitIds": "Nonzero character group slots, in source slot order.", + "preAction": "The source pre-action row, or null; its tweet is resolved separately.", + "fixture": "This family has no fixture anchor; it requires common multi-character runtime support.", + }, + "talks": rows, "summary": summary, + }, summary diff --git a/src/chara/communication_actions.py b/src/chara/communication_actions.py new file mode 100644 index 0000000..719ccda --- /dev/null +++ b/src/chara/communication_actions.py @@ -0,0 +1,107 @@ +"""Per-character communication performances, distinct from spoken dialogue. + +The script's lifetime loop and participant-gated random prefix are retained. +Channel steps use the existing performance reader and its source library's +argument order. No dialogue, voice or translated text is invented here. +""" +from collections import Counter +import re + +from core.jsonio import write_json +from core.master import Master +from chara import alone_actions +from chara.talks import _asset_named, _calls, _resolve, _strip_comments, _text_assets + +PACKAGE = "mysekai__some_character_communication" +TABLE = "mysekaiCharacterTalkTogetherCommunications" + + +def parse_program(script, tables, scalars, defaults): + text = _strip_comments(script).strip() + loop = re.fullmatch(r"while\s+enable_communication\(\)\s+do\s*(.*?)\s*end", text, re.S) + if not loop: + raise ValueError("unsupported communication lifetime loop") + body = loop[1] + branches, random = [], None + prefix = re.match(r"\s*rand\s*=\s*math\.random\(0,\s*99\)\s*", body) + if prefix: + random = {"min": 0, "max": 99} + body = body[prefix.end():] + branch = re.match(r"if\s+is_communicating\((\w+\.\w+)\)\s+and\s+rand\s*<\s*(\d+)\s+then(.*?)" + r"elseif\s*\(\s*rand\s*<\s*(\d+)\s*\)\s*then(.*?)\bend\b", body, re.S) + if not branch: + raise ValueError("unsupported communication random branch") + participant, _ = _resolve(branch[1], tables, scalars) + if not isinstance(participant, int): + raise ValueError("unresolved communication participant") + branches = [{"condition": {"participant": participant, "randomBelow": int(branch[2])}, "body": branch[3]}, + {"condition": {"participant": None, "randomBelow": int(branch[4])}, "body": branch[5]}] + body = body[branch.end():] + + def steps(fragment): + if re.search(r"\b(?:if|elseif|else|end|while|for|repeat|function|return)\b", fragment): + raise ValueError("unhandled communication control flow") + allowed = set(alone_actions.CALL_OPS) | {"wait_time"} + calls = list(_calls(fragment)) + # _calls deliberately scans the shared talk vocabulary. Also account + # for every lexical function name so a future command cannot vanish. + names = re.findall(r"\b([A-Za-z_]\w*)\s*\(", fragment) + if any(name not in allowed for name in names) or len(names) != len(calls): + raise ValueError("unsupported communication operation") + result, elapsed = [], 0.0 + for op, args in calls: + call = ("alone_wait_time(nil, " + ",".join(args) + ")" if op == "wait_time" + else op + "(" + ",".join(args) + ")") + emitted, count = alone_actions._steps(call, tables, scalars, defaults) + if count != 1: + raise ValueError("communication call accounting mismatch") + if op != "wait_time": + who, _ = _resolve(args[0], tables, scalars) + if not isinstance(who, int): + raise ValueError("unresolved communication actor") + for step in emitted: + step["who"] = who + for step in emitted: + step["t"] = elapsed + if step["op"] == "animation" and not isinstance(step["motion"], str): + raise ValueError("unresolved communication motion") + if step["op"] == "wait": + if not isinstance(step["seconds"], (int, float)) or step["seconds"] < 0: + raise ValueError("invalid communication wait duration") + elapsed += step["seconds"] + result.append(step) + return result + + for branch in branches: + branch["steps"] = steps(branch.pop("body")) + tail = steps(body) + if not any(step["op"] == "wait" and isinstance(step["seconds"], (int, float)) and step["seconds"] > 0 + for step in tail): + raise ValueError("communication lifetime loop has no positive wait") + return {"kind": "whileCommunication", "random": random, "branches": branches, "steps": tail} + + +def extract_communication_actions(master_source, bundle, out_path, master_cache=None): + assets = _text_assets(bundle) + defines = _asset_named(assets, "defines.lua") + library = _asset_named(assets, "lib.lua") + if defines is None or library is None: + raise ValueError("communication package is missing its libraries") + tables = alone_actions.parse_constant_tables(defines) + scalars = alone_actions.parse_constant_scalars(defines, tables) + defaults = alone_actions.parse_library_defaults(library) + units, operations = {}, Counter() + for row in Master(master_source, cache_dir=master_cache).table(TABLE): + unit = str(row["gameCharacterUnitId"]) + if unit in units: + raise ValueError("duplicate communication character") + script = _asset_named(assets, row["lua"] + ".lua") + if script is None: + raise ValueError("missing communication script: " + row["lua"]) + program = parse_program(script, tables, scalars, defaults) + units[unit] = {"id": row["id"], "lua": row["lua"], "program": program} + for steps in [program["steps"], *(b["steps"] for b in program["branches"])]: + operations.update(step["op"] for step in steps) + summary = {"units": len(units), "steps": sum(operations.values()), "operations": dict(operations)} + write_json(out_path, {"version": 1, "units": units, "summary": summary}) + return {**summary, "path": str(out_path)} diff --git a/src/chara/talks.py b/src/chara/talks.py index eb120fc..6e3322b 100644 --- a/src/chara/talks.py +++ b/src/chara/talks.py @@ -610,7 +610,9 @@ def parse_script(text, tables, scalars=None, script_name=None, defaults=None, def _text_assets(bundle): """Return ``{TextAsset name: decoded script text}`` from a Unity bundle.""" out = {} - for obj in UnityPy.load(bundle).objects: + # These small script bundles are consumed in one pass. Loading their + # bytes avoids retaining an open source file through Unity object cycles. + for obj in UnityPy.load(Path(bundle).read_bytes()).objects: if obj.type.name != "TextAsset": continue tree = obj.read_typetree() @@ -911,6 +913,12 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, # its exact serialized form. birthday, birthday_report = birthday_talks(master, assets, tables, scalars) doc["birthdayParty"] = birthday + # Multi-character ordinary talks share the source package and parser, + # but have their own membership and runtime plan. Do not put them into + # any participant's solo lottery or invent a fixture for them. + from chara.common_talks import common_talks + common, common_report = common_talks(master, assets, tables, scalars) + doc["commonSomeCharacter"] = common os.makedirs(os.path.dirname(os.path.abspath(out_path)) or ".", exist_ok=True) with open(out_path, "w", encoding="utf-8", newline="\n") as handle: json.dump(doc, handle, ensure_ascii=False, indent=1, allow_nan=False) @@ -921,6 +929,7 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, "constantSource": constant_source, "constantTables": {key: len(value) for key, value in tables.items()}, "birthdayParty": birthday_report, + "commonSomeCharacter": common_report, } diff --git a/src/chara/text_catalog.py b/src/chara/text_catalog.py new file mode 100644 index 0000000..ddd88d5 --- /dev/null +++ b/src/chara/text_catalog.py @@ -0,0 +1,260 @@ +"""Versioned display-text catalogues, independent of executable scripts. + +Text IDs survive source revisions through unique voice/context anchors or +unchanged text. Ambiguous repetitions are never silently paired after an +edit. Bindings locate a display operation in a specific script revision; +they are not the translation identity. +""" +from collections import defaultdict +import hashlib +import json + +from core.jsonio import write_json + +SCHEMA = "moly-text-source/1" +REGIONS = {"jp", "cn", "tw", "en", "kr"} + + +def digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest() + + +def corpus_rows(document): + for unit, group in (document.get("units") or {}).items(): + for row in group.get("talks", []): + yield "general", row, [int(unit)] + for row in document.get("talks", []): + yield "fixture", row, row["unitIds"] + for row in (document.get("commonSomeCharacter") or {}).get("talks", []): + yield "common", row, row["unitIds"] + for lua, row in (document.get("birthdayParty") or {}).get("talks", {}).items(): + yield "birthday", {**row, "lua": lua, "talkId": lua}, [] + + +def _display_rows(row): + """Original label tokens remain untouched; their displays get bindings.""" + rows = [] + speaker, unit, pending = "", None, [] + for index, step in enumerate(row["steps"]): + op = step["op"] + if op == "label": + speaker = step["name"] + unit, pending = None, [] + rows.append({"kind": "speaker", "step": index, "text": speaker, + "speaker": speaker, "speakerUnit": None, "cues": []}) + elif op in ("voice", "fixture_voice"): + unit = step.get("who", step.get("fixture")) + pending.append(step["cue"]) + # Labels conventionally precede voice; attach its identity to + # that label's display without changing the label/control token. + for display in reversed(rows): + if display["kind"] == "speaker": + display["speakerUnit"] = unit + display["cues"] = list(pending) + break + if display["kind"] == "text": + break + elif op == "text": + rows.append({"kind": "text", "step": index, "text": step["text"], + "speaker": speaker, "speakerUnit": unit, "cues": list(pending)}) + pending = [] + for i, display in enumerate(rows): + if not isinstance(display["text"], str): + raise ValueError(f"{row['lua']}: display text is not a string") + display["position"] = i + display["sourceHash"] = digest([display["kind"], display["text"], display["speaker"], + display["speakerUnit"], display["cues"]]) + display["anchor"] = digest([display["kind"], display["speakerUnit"], display["cues"], + display.get("tweetId")]) if display["cues"] or display["kind"] == "tweet" else None + return rows + + +def _assign_ids(region, script, current, previous): + old = previous.get("lines", []) if previous else [] + retired_before = previous.get("retiredLines", []) if previous else [] + used, matched = set(), {} + # Unchanged complete display sequence permits exact occurrence mapping. + # A motion insertion changes bindings but not this sequence. + if [r["sourceHash"] for r in old] == [r["sourceHash"] for r in current]: + matched = {i: r for i, r in enumerate(old)} + used = {r["id"] for r in old} + else: + for field in ("anchor", "sourceHash"): + old_by, new_by = defaultdict(list), defaultdict(list) + for r in old: + if r["id"] not in used and r.get(field): + old_by[r[field]].append(r) + for i, r in enumerate(current): + if i not in matched and r.get(field): + new_by[r[field]].append(i) + for key, indices in new_by.items(): + if len(indices) == 1 and len(old_by.get(key, [])) == 1: + match = old_by[key][0] + matched[indices[0]] = match + used.add(match["id"]) + old_ids = {r["id"] for r in old + retired_before} + allocated = set(old_ids) + for i, row in enumerate(current): + if i in matched: + prior = matched[i] + row["id"] = prior["id"] + row["change"] = "unchanged" if row["sourceHash"] == prior["sourceHash"] else "changed" + else: + salt = 0 + while True: + identity = "t_" + digest([region, script, row["kind"], row["anchor"], + row["sourceHash"], i, salt])[:32] + if identity not in allocated: + break + salt += 1 + row["id"] = identity + row["change"] = "new" + allocated.add(identity) + return retired_before + [r for r in old if r["id"] not in used] + + +def build_catalog(documents, *, region, source_revision, previous=None): + if region not in REGIONS or not source_revision: + raise ValueError("a supported source region and explicit source revision are required") + if previous and (previous.get("schema") != SCHEMA or previous.get("region") != region): + raise ValueError("previous text catalogue belongs to another schema or region") + old = {r["key"]: r for r in (previous or {}).get("retiredScripts", [])} + old.update({r["key"]: r for r in (previous or {}).get("scripts", [])}) + scripts, tweet_refs = {}, {} + for document in documents: + for kind, row, members in corpus_rows(document): + key = row["lua"].removesuffix(".lua") + lines = _display_rows(row) + signature = digest(row["steps"]) + content = {"kind": kind, "id": row["talkId"], "unitIds": members} + tweet = row.get("tweet") + if tweet and isinstance(tweet.get("text"), str): + tweet_key = "@tweet/" + str(tweet["id"]) + if tweet_key in tweet_refs and tweet_refs[tweet_key]["tweet"] != tweet: + raise ValueError(f"conflicting source bodies for tweet {tweet['id']}") + ref = tweet_refs.setdefault(tweet_key, {"tweet": tweet, "contents": []}) + tweet_content = {**content, "scriptKey": key} + if tweet_content not in ref["contents"]: + ref["contents"].append(tweet_content) + content = {**content, "tweetKey": tweet_key} + if key in scripts: + existing = scripts[key] + if existing["scriptHash"] != signature: + raise ValueError(f"conflicting source bodies for script {key}") + if content not in existing["contents"]: + existing["contents"].append(content) + continue + retired = _assign_ids(region, key, lines, old.get(key)) + scripts[key] = {"key": key, "scriptHash": signature, "contents": [content], + "lines": lines, "retiredLines": retired, + "displaySequence": [s["op"] for s in row["steps"] if s["op"] in + ("label", "text", "voice", "fixture_voice", "wait_click")], + "speakerIdentity": sorted({str(s.get("who", s.get("fixture"))) + for s in row["steps"] + if s["op"] in ("voice", "fixture_voice")})} + # A script may be referenced by several talk rows with different tweets. + # Tweets have their own master identities, not a script-body identity. + for key, ref in tweet_refs.items(): + tweet = ref["tweet"] + lines = [{"kind": "tweet", "step": None, "tweetId": tweet["id"], + "text": tweet["text"], "speaker": "", "speakerUnit": None, + "cues": [], "position": 0, + "sourceHash": digest(["tweet", tweet]), + "anchor": digest(["tweet", tweet["id"]])}] + retired = _assign_ids(region, key, lines, old.get(key)) + scripts[key] = {"key": key, "scriptHash": digest(tweet), + "contents": ref["contents"], "lines": lines, "retiredLines": retired, + "tweetContext": {k: v for k, v in tweet.items() if k != "text"}} + result = {"schema": SCHEMA, "region": region, "sourceRevision": source_revision, + "scripts": list(scripts.values()), + "retiredScripts": [r for key, r in old.items() if key not in scripts]} + # Import diagnostics (new/changed/unchanged and retired history) are not + # published source content. Reimporting identical bytes keeps a revision. + result["revision"] = digest({ + "schema": SCHEMA, "region": region, "sourceRevision": source_revision, + "scripts": [{k: ([{lk: lv for lk, lv in line.items() if lk != "change"} + for line in v] if k == "lines" else v) + for k, v in script.items() if k != "retiredLines"} + for script in result["scripts"]], + }) + return result + + +def match_official(source, donor, *, locale): + """Pair complete display sequences only when spoken context agrees. + + Same script name is a candidate, never sufficient proof. Require equal + participating units, line kinds and per-line voice/speaker identity. + Lines without a voice anchor require an explicit reviewed mapping. + """ + expected = {"zh-CN": "cn", "zh-TW": "tw", "en-US": "en", "ko-KR": "kr"} + if source.get("schema") != SCHEMA or donor.get("schema") != SCHEMA: + raise ValueError("unsupported source catalogue schema") + if expected.get(locale) != donor["region"]: + raise ValueError("official text locale does not match donor region") + donors = {r["key"]: r for r in donor["scripts"]} + entries, issues = [], [] + for script in source["scripts"]: + other = donors.get(script["key"]) + reason = None + if other is None: + reason = "missing_script" + else: + def members(value): + return sorted({tuple(c["unitIds"]) for c in value["contents"]}) + def spoken(value): + return [(r["kind"], r["speakerUnit"], r["cues"]) for r in value["lines"] if r["kind"] != "tweet"] + if members(script) != members(other): + reason = "participants_differ" + elif script.get("displaySequence") != other.get("displaySequence"): + reason = "display_control_sequence_differs" + elif script.get("speakerIdentity") != other.get("speakerIdentity"): + reason = "speaker_identities_differ" + elif script.get("tweetContext") != other.get("tweetContext"): + reason = "tweet_context_differs" + elif script.get("tweetContext") is not None and sorted( + {c["scriptKey"] for c in script["contents"]} + ) != sorted({c["scriptKey"] for c in other["contents"]}): + reason = "tweet_script_links_differ" + elif spoken(script) != spoken(other): + reason = "display_sequence_differs" + if reason: + issues.append({"script": script["key"], "reason": reason}) + continue + source_lines = script["lines"] + donor_lines = other["lines"] + anchors = defaultdict(int) + for line in source_lines: + anchors[line.get("anchor")] += 1 + for line, official in zip(source_lines, donor_lines): + if line.get("anchor") and anchors[line["anchor"]] > 1: + issues.append({"script": script["key"], "id": line["id"], "reason": "ambiguous_voice_anchor"}) + continue + if line["kind"] != "tweet" and (not line["cues"] or line["speakerUnit"] is None): + issues.append({"script": script["key"], "id": line["id"], "reason": "unanchored_line"}) + continue + if not official["text"].strip() or official["text"] == line["text"]: + issues.append({"script": script["key"], "id": line["id"], "reason": "empty_or_unchanged_text"}) + continue + entries.append({"id": line["id"], "sourceHash": line["sourceHash"], + "text": official["text"], "source": "official_" + donor["region"], + "donor": {"region": donor["region"], "revision": donor["revision"], + "script": other["key"], "id": official["id"], + "sourceHash": official["sourceHash"]}}) + result = {"schema": "moly-text-overlay/1", "sourceRegion": source["region"], + "sourceRevision": source["revision"], "locale": locale, + "entries": entries, "issues": issues} + result["revision"] = digest(result) + return result + + +def export_catalog(paths, out, *, region, source_revision, previous=None): + def read(path): + with open(path, encoding="utf-8") as handle: + return json.load(handle) + result = build_catalog([read(p) for p in paths], region=region, + source_revision=source_revision, previous=read(previous) if previous else None) + write_json(out, result) + return {"scripts": len(result["scripts"]), + "lines": sum(len(r["lines"]) for r in result["scripts"]), "revision": result["revision"]} diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..f84057b --- /dev/null +++ b/src/cli/__init__.py @@ -0,0 +1 @@ +"""Command groups; library layers do not import the command line.""" diff --git a/src/cli/__main__.py b/src/cli/__main__.py new file mode 100644 index 0000000..774cc25 --- /dev/null +++ b/src/cli/__main__.py @@ -0,0 +1,3 @@ +from .main import main + +raise SystemExit(main()) diff --git a/src/cli/legacy.py b/src/cli/legacy.py new file mode 100644 index 0000000..9371267 --- /dev/null +++ b/src/cli/legacy.py @@ -0,0 +1,1721 @@ +"""Single command-line entry point for bundle extraction.""" +import argparse +import os +import warnings +import json +import sys +from pathlib import Path + +# Extraction output root; MOLY_EXTRACT_OUT lets a workspace keep it outside the checkout. +DEFAULT_DATA_ROOT = os.environ.get("MOLY_EXTRACT_OUT") or "local-data" + + +PLAYER_DATA_HELP = ("the player data: the container file (assets/bin/Data/data.unity3d), " + "or, for a build without one, each APK whose assets/bin/Data members " + "are its loose player files; repeat for the base APK and the asset pack") + + +def _configure_unity(fallback_version): + import UnityPy.config + if fallback_version: + UnityPy.config.FALLBACK_UNITY_VERSION = fallback_version + + +def _master_source(args): + """Directory or base URL for the caller's master tables, plus a cache dir. + + ``--master`` names a local directory; ``--master-url`` fetches each table by + appending its name to a base URL (with no value, the public default base). + Neither is implied: without one of them nothing is read and nothing is + fetched. + """ + from core.master import DEFAULT_MASTER_URL + url = getattr(args, "master_url", None) + if url is not None: + return url or DEFAULT_MASTER_URL, getattr(args, "master_cache", None) + return getattr(args, "master", None), None + + +def _write_programs(directory, entries, platform=None): + """Write each program body to its own file; return how many were written. + + A body is named by shader, platform and record index rather than by content + hash, so a caller reading a file knows which variant produced it; identical + variants therefore appear more than once, which is the honest shape -- the + census reports the distinct count separately. + """ + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + written = 0 + for entry in entries: + stem = "".join(c if c.isalnum() or c in "-_" else "_" for c in entry["shader"]) + for block in entry["platforms"]: + if platform is not None and block["platform"] != platform: + continue + for row in block["programs"]: + suffix = "glsl" if row["isText"] else "bin" + name = f"{stem}__platform{block['platform']}__record{row['record']}.{suffix}" + (directory / name).write_bytes(row.get("code") or b"") + written += 1 + return written + + +def _extract_failed(report): + """A failed derived artifact is a failed extraction, not a skipped option. + + So is a failed player-data artifact: its input was supplied and it could + not be read, which is a failure of the run, while a player-data artifact + whose input nobody supplied stays ``skipped`` and is named as an input gap. + """ + return bool(report["summary"]["failed"]) or any( + entry.get("status") == "failed" + for entry in report.get("derived", []) + report.get("playerData", [])) + + +def _print_extract_summary(report, out_dir): + s = report["summary"] + print(f"extracted {s['succeeded']}/{s['requested']} bundles -> {out_dir} " + f"(failed {s['failed']}, unsupported {s['unsupported']})") + for entry in report["bundles"]: + if entry["status"] == "failed": + print(f" failed: {entry['bundle']}: {entry['error']}") + for entry in report.get("derived", []) + report.get("playerData", []): + if entry.get("status") == "failed": + print(f" failed artifact: {entry['artifact']}: {entry.get('error', '')}") + programs = entry.get("sourcePrograms") + if programs: + print(f" particle source programs: {programs['status']} " + f"({programs['converted']}/{programs['gles3Variants']} GLES3 variants converted, " + f"unavailable {programs['unavailable']})") + print(f" {programs['note']}") + # An artifact whose input nobody supplied is not a failure of the run, but + # it is not silent either: each one is named with its reason (a failed one + # is already named above). + for gap in (report.get("inputGaps") or {}).get("unaccepted", []): + if gap.get("why") != "failed": + print(f" input gap: {gap['artifact']}: {gap.get('detail', '')}") + print(f"report: {report['report']}") + if str(out_dir) == "local-data": + print("view: python -m http.server 8000 (from the repository root), then open\n" + " http://localhost:8000/examples/viewer/index.html?base=../../local-data") + else: + print("view: serve the repository root over HTTP and open " + "examples/viewer/index.html?base=") + + +def main(argv=None): + from chara.rotation_encoding import DEFAULT_ROTATION_ENCODING, ROTATION_ENCODINGS + # UnityPy warns about asset shapes it does not model. Silencing that is + # this command's choice to make, so it is made here, once, for this + # process. It used to sit at module scope in four extractors, where + # importing one rewrote the warning filters of any program that used this + # package as a library -- including filters that program had set itself. + warnings.filterwarnings("ignore") + ap = argparse.ArgumentParser(prog="moly", description="Extract Unity humanoid assets", + epilog="Grouped workflows: moly source --help; moly export talks --help; moly export text-runtime --help; moly depot --help. Flat exports remain compatible.") + ap.add_argument("--unity-version", default=None) + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("pull", help="download, decrypt, and extract character asset packs") + p.add_argument("--manifest", required=True) + p.add_argument("--out", help="download/decrypt workspace (default: moly-pull-output)") + p.add_argument("--extract-out", default=None, + help="where extraction artifacts go (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") + p.add_argument("--json", action="store_true", + help="print the full JSON report to stdout instead of a summary") + p.add_argument("--asset-base-url", required=True) + p.add_argument("--workers", type=int, default=8) + p.add_argument("--retries", type=int, default=4) + p.add_argument("--master", help="directory of caller-supplied master tables") + p.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append .json to; no value uses the public default base") + p.add_argument("--master-cache", help="where fetched tables are cached") + p.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + p.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") + x = sub.add_parser("extract", help="extract bundles listed in a manifest") + x.add_argument("--manifest", + help="optional bundle list; omitted = select every supported " + "asset-pack bundle found under --bundles") + x.add_argument("--bundles", required=True) + x.add_argument("--out", default=DEFAULT_DATA_ROOT, + help="output directory (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") + x.add_argument("--json", action="store_true", + help="print the full JSON report to stdout instead of a summary") + x.add_argument("--master", help="directory of caller-supplied master tables") + x.add_argument("--player-data", action="append", + help="the APK player data: the container file " + "(assets/bin/Data/data.unity3d), or, for a build " + "without one, each APK whose assets/bin/Data members " + "are its loose player files (repeat: the base APK and " + "the asset pack). It supplies the built-in UI and " + "screen-layer assets that the download path does not " + "carry (container builds only) and the Resources " + "objects: the live transitioner, the effect type " + "table, the graphics configuration, the layout grid") + x.add_argument("--player-types", metavar="DIR", + help="managed assembly stubs of the same player build (a DummyDll " + "directory); the Resources objects read from --player-data " + "(the live transitioner, the effect type table, the drop-item " + "shading) need them, and without them those artifacts are " + "skipped with the reason") + x.add_argument("--player-declarations", metavar="FILE", + help="the same build's declaration dump; names the effect types in " + "effect-resources/ (without it the names are null)") + x.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + x.add_argument("--master-cache", help="where fetched tables are cached") + x.add_argument("--fixture-meshes", action="store_true", + help="also write furniture geometry (fixture-models/), one " + "glTF binary per package; without it that pass is " + "skipped and the passes that need the geometry report " + "it as a missing dependency") + x.add_argument("--fixture-particles", action="store_true", + help="also write furniture particle emitters " + "(fixture-particles/), one document per package " + "across the fixture-interface, cutscene-timeline and " + "fixture-timeline families; without it that pass is " + "skipped. Playable output requires --shader-compiler; " + "without it only source data is saved, not playable assets") + x.add_argument("--shader-compiler", metavar="PATH", + help="pinned offline source shader compiler required for playable " + "furniture particles; omitting it saves source-only data") + x.add_argument("--builtin-resources", action="append", default=[], + metavar="PATH", + help="the engine's own built-in resource container, or a " + "directory holding it; some particle-system renderers " + "and some phenomena emitters draw a copy of a " + "built-in primitive mesh and no package ships those. " + "Repeatable. Without it those mesh slots stay listed " + "as unresolved in fixture-particles/ and in the " + "phenomena index, exactly as before") + x.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + x.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") + x.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") + c = sub.add_parser("characters", help="extract one character bundle") + c.add_argument("--bundle", required=True); c.add_argument("--out-dir", required=True); c.add_argument("--name", required=True) + c.add_argument("--aux-bundle", action="append", default=[]); c.add_argument("--motion-bundle"); c.add_argument("--clips"); c.add_argument("--unit-id", type=int); c.add_argument("--facial-tables"); c.add_argument("--atlas-cell", default="512x256"); c.add_argument("--no-cloth", action="store_true") + m = sub.add_parser("motion-library", help="export the shared humanoid motion library") + m.add_argument("--reference-bundle", required=True); m.add_argument("--motion-bundle", required=True); m.add_argument("--aux-bundle", action="append", default=[]); m.add_argument("--clips"); m.add_argument("--out-dir", required=True); m.add_argument("--name", default="motion-library") + m.add_argument("--controller-bundle", action="append", default=None, + help="character package whose AnimatorController selects the clips " + "(by serialized identity); repeat per package. Without it the " + "first clip of each name is kept and duplicates are listed") + m.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="rotation key storage (default: snorm16); float32 disables lossy encoding") + t = sub.add_parser("facial-tables", help="dump facial tables") + t.add_argument("--bundle", required=True); t.add_argument("--out", required=True) + n = sub.add_parser("alone-actions", help="dump per-character performance data (motion x facial x timing)") + n.add_argument("--bundle", required=True); n.add_argument("--out", required=True) + g = sub.add_parser("registry", help="build the character registry from master tables") + g.add_argument("--master", help="directory of caller-supplied master tables") + g.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + g.add_argument("--master-cache", help="where fetched tables are cached") + g.add_argument("--manifest", required=True, help="manifest naming the character bundles") + g.add_argument("--motion-index", help="shared motion library index, to check motion names") + g.add_argument("--out", required=True) + + k = sub.add_parser("talks", help="extract ordinary solo and multi-character talk scripts") + k.add_argument("--master", help="directory of caller-supplied master tables") + k.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + k.add_argument("--master-cache", help="where fetched tables are cached") + k.add_argument("--bundle", required=True, help="talk scenario bundle") + k.add_argument("--lib", help="decrypted lib package carrying the constant " + "tables the scripts name; without it every " + "Table.key token stays a source token") + k.add_argument("--out", required=True) + + tc = sub.add_parser("text-catalog", help="export versioned display-text identities for translation") + tc.add_argument("--corpus", action="append", required=True, help="extracted talk document; repeatable") + tc.add_argument("--region", required=True, choices=("jp", "cn", "tw", "en", "kr")) + tc.add_argument("--source-revision", required=True, help="explicit source resource revision") + tc.add_argument("--previous", help="previous same-region text catalogue; preserves stable IDs") + tc.add_argument("--out", required=True) + ot = sub.add_parser("official-text", help="match official display text from another source region") + ot.add_argument("--source", required=True, help="source text catalogue") + ot.add_argument("--donor", required=True, help="official regional text catalogue") + ot.add_argument("--locale", required=True, choices=("zh-CN", "zh-TW", "en-US", "ko-KR")) + ot.add_argument("--out", required=True) + + tw = sub.add_parser("tweets", help="extract tweet texts and per-character after-edit pools") + tw.add_argument("--master", help="directory of caller-supplied master tables") + tw.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + tw.add_argument("--master-cache", help="where fetched tables are cached") + tw.add_argument("--out", required=True) + + tt = sub.add_parser("tweet-tables", + help="extract the tweet selection tables (greetings, " + "greeting conditions, site entries, talk pre-actions)") + tt.add_argument("--master", help="directory of caller-supplied master tables") + tt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + tt.add_argument("--master-cache", help="where fetched tables are cached") + tt.add_argument("--out", required=True) + + sg = sub.add_parser("site-groups", + help="extract the mysekai site-group table " + "(siteGroupId -> the site ids it holds)") + sg.add_argument("--master", help="directory of caller-supplied master tables") + sg.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + sg.add_argument("--master-cache", help="where fetched tables are cached") + sg.add_argument("--out", required=True) + + bp = sub.add_parser("birthday-parties", + help="extract the birthday-party campaign table the " + "site-map festival gate reads (startAt/closedAt windows)") + bp.add_argument("--master", help="directory of caller-supplied master tables") + bp.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bp.add_argument("--master-cache", help="where fetched tables are cached") + bp.add_argument("--out", required=True) + + bd = sub.add_parser("birthday-party-delivery", + help="extract the birthday-party delivery tables the " + "delivery site reads (rewards, point bonuses, total rewards)") + bd.add_argument("--master", help="directory of caller-supplied master tables") + bd.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bd.add_argument("--master-cache", help="where fetched tables are cached") + bd.add_argument("--out", required=True) + + gv = sub.add_parser("gate-visit-policy", + help="extract the gate visit tables as policy data for the " + "server mock (not a client character input)") + gv.add_argument("--master", help="directory of caller-supplied master tables") + gv.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + gv.add_argument("--master-cache", help="where fetched tables are cached") + gv.add_argument("--out", required=True) + + tm = sub.add_parser("time-manager", + help="extract the project TimeManager settings (fixed step, " + "maximum allowed timestep, time scale, particle step) " + "from an APK or a globalgamemanagers file") + tm.add_argument("--source", required=True, + help="an APK, or a player-data globalgamemanagers file") + tm.add_argument("--out", required=True) + + cc = sub.add_parser("client-config", + help="extract the ClientConfig deliverable panel " + "(the four typed dictionaries, keyed by id)") + cc.add_argument("--master", help="directory of caller-supplied master tables") + cc.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + cc.add_argument("--master-cache", help="where fetched tables are cached") + cc.add_argument("--out", required=True) + + from core.master_tables import REGISTERED_TABLES + mt = sub.add_parser("master-tables", + help="extract selected key-addressed mysekai master " + "tables, defaulting to blueprints, items, music " + "records and wordings") + mt.add_argument("--master", help="directory of caller-supplied master tables") + mt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + mt.add_argument("--master-cache", help="where fetched tables are cached") + mt.add_argument("--table", dest="tables", action="append", + choices=tuple(REGISTERED_TABLES), + help="registered table to export; repeat to select more " + "(omitting this exports only the original four tables)") + mt.add_argument("--out", required=True, + help="directory the selected documents are written into") + + e = sub.add_parser("emoticons", help="extract overhead-item effect packages") + e.add_argument("--bundle", action="append", required=True); e.add_argument("--out-dir", required=True) + v = sub.add_parser("avatar-parts", help="extract player-appearance packages (skin/decoration/penlight)") + v.add_argument("--bundle", action="append", required=True); v.add_argument("--out-dir", required=True) + ht = sub.add_parser("harvest-tools", help="extract original harvest-tool prefabs and their material data") + ht.add_argument("--bundle", action="append", required=True) + ht.add_argument("--bundle-root", help="directory containing declared shader dependencies") + ht.add_argument("--out-dir", required=True, help="tool directory, normally /avatar/tools") + ptm = sub.add_parser("player-tool-models", + help="extract the avatar's hand-item prefabs and their material data " + "(mysekai/player_tool_model/*)") + ptm.add_argument("--bundle", action="append", required=True) + ptm.add_argument("--bundle-root", help="directory containing declared shader dependencies") + ptm.add_argument("--out-dir", required=True, + help="model directory, normally /avatar/player-tool-models") + er = sub.add_parser("effect-resources", + help="export the effect manager's type table " + "(Resources Mysekai/Effects/EffectResourceData) from player data") + er.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + er.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + er.add_argument("--declarations", + help="the same build's declaration dump, for effect type names") + er.add_argument("--bundle-root", + help="bundle directory to check each row's bundle and prefab against") + er.add_argument("--out", required=True, help="output JSON path") + dis = sub.add_parser("drop-item-shade", + help="export the drop-item shading settings of the graphics " + "configuration (Resources Graphics/Data/MysekaiGraphicsConfig)") + dis.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + dis.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + dis.add_argument("--out", required=True, help="output JSON path") + sts = sub.add_parser("stencil-shadow", + help="export the character stencil-shadow colour and the material " + "rendering settings of the graphics configuration") + sts.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + sts.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + sts.add_argument("--out", required=True, help="output JSON path") + lgr = sub.add_parser("layout-grid", + help="export the layout editor's placement grid: its settings, " + "materials, shader programs and colour profiles " + "(Resources Data/MysekaiGridSettings and " + "Data/GridColorProfileTable)") + lgr.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + lgr.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + lgr.add_argument("--out-dir", required=True, + help="output directory, normally /layout-grid") + rpf = sub.add_parser("resource-prefab", + help="extract one prefab the player data ships in Resources, by load name") + rpf.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + rpf.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + rpf.add_argument("--load-name", required=True, help="the Resources.Load path") + rpf.add_argument("--out-dir", required=True, + help="directory receiving /.{json,glb} and textures") + rpf.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", + help="the engine's built-in resource file (unity default resources, " + "unity_builtin_extra) or a directory holding them; a container " + "build keeps them beside data.unity3d; repeat for each") + rpr = sub.add_parser("resource-programs", + help="write the compiled programs of every shader a Resources " + "prefab's materials use (the program library layout)") + rpr.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + rpr.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + rpr.add_argument("--load-name", required=True, help="the Resources.Load path") + rpr.add_argument("--out-dir", required=True, + help="directory receiving /shaders/ (the resource-prefab out-dir)") + rpr.add_argument("--shader-compiler", + help="offline shader compiler executable; without it the programs are " + "written as source only (no conversions)") + fp = sub.add_parser("fixture-particles", + help="write particle source-schema documents (the fixture-particles " + "layout) for explicitly named bundles") + fp.add_argument("--bundle", action="append", required=True) + fp.add_argument("--bundle-root", help="directory the bundles' dependencies are read from") + fp.add_argument("--domain", action="append", + help="routed domain to extract; repeat; default the three furniture domains") + fp.add_argument("--shader-compiler", metavar="PATH", + help="pinned offline shader compiler; without it programs stay source-only") + fp.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", + help="the engine's built-in resource container (or its directory), " + "for renderers that point at a built-in mesh") + fp.add_argument("--out-dir", required=True) + pp = sub.add_parser("prefab-package", + help="extract one named prefab of a bundle with the site package reader") + pp.add_argument("--bundle", required=True, help="the bundle file") + pp.add_argument("--prefab", required=True, + help="the prefab's container basename, e.g. fieldcamera.prefab") + pp.add_argument("--bundle-root", help="directory the bundle's dependencies are read from") + pp.add_argument("--out-dir", required=True, + help="directory receiving /.{json,glb} and textures") + sae = sub.add_parser("site-action-effects", + help="extract the site-move and harvest-action effect prefabs " + "(mysekai/effect/site/harvest/action/*)") + sae.add_argument("--bundle", action="append", required=True) + sae.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + sae.add_argument("--out-dir", required=True, + help="effect directory, normally /site-action-effects") + she = sub.add_parser("site-home-effects", + help="extract the home-site effect prefabs, the craft effect among " + "them (mysekai/effect/site/home/*)") + she.add_argument("--bundle", action="append", required=True) + she.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + she.add_argument("--out-dir", required=True, + help="effect directory, normally /site-home-effects") + fpe = sub.add_parser("fixture-put-effects", + help="extract the furniture placement effect prefab " + "(mysekai/effect/fixture/fx_fixture_put_01)") + fpe.add_argument("--bundle", action="append", required=True) + fpe.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + fpe.add_argument("--out-dir", required=True, + help="effect directory, normally /fixture-put-effects") + ssw = sub.add_parser("site-sound-wave", + help="resolve MysekaiSiteView._soundWaveMeshFilter (the footstep " + "surface) of site scene packages: node, mesh and vertex colours") + ssw.add_argument("--bundle", action="append", required=True, + help="a mysekai__site__field__ package; repeat") + ssw.add_argument("--bundle-root", help="directory the scene packages' dependencies are read from") + ssw.add_argument("--out", required=True, help="output JSON path") + sce = sub.add_parser("site-common-action-effects", + help="extract the player's foot effect prefabs " + "(mysekai/effect/site/common/action/*)") + sce.add_argument("--bundle", action="append", required=True) + sce.add_argument("--bundle-root", + help="directory containing declared effect, texture and shader dependencies") + sce.add_argument("--out-dir", required=True, + help="effect directory, normally /site-common-action-effects") + sho = sub.add_parser("site-harvest-object-effects", + help="extract the harvest-object hit, delete and stay effect prefabs " + "(mysekai/effect/site/harvest/object//*)") + sho.add_argument("--bundle", action="append", required=True) + sho.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + sho.add_argument("--out-dir", required=True, + help="effect directory, normally /site-harvest-object-effects") + pa = sub.add_parser("player-avatar", + help="export the player avatar composite (audience model + skeleton + motion clips) as one glb") + pa.add_argument("--bundle", action="append", required=True, + help="the avatar model bundle first, then every motion bundle whose clips " + "are to be resolved against it (the order export_player_avatar takes); " + "repeat --bundle per package") + pa.add_argument("--out-dir", required=True) + pa.add_argument("--name", default="mysekai__player_avatar", + help="output basename (glb + .index.json + .motion-manifest.json)") + w = sub.add_parser("phenomena", help="extract weather (phenomena) environment packages") + w.add_argument("--bundle", action="append", required=True, + help="an environment package, or the shared phenomena thumbnail package") + w.add_argument("--bundle-root", + help="directory the packages a bundle depends on are read from") + w.add_argument("--builtin-resources", action="append", default=[], + metavar="PATH", + help="the engine's own built-in resource container, or a " + "directory holding it; some emitters draw copies of a " + "built-in primitive and no package ships those. Repeatable. " + "Without it those meshes stay listed under `unsupported`, " + "exactly as before") + w.add_argument("--shader-compiler", metavar="PATH", + help="moly-shader-bridge executable; without it source programs " + "are preserved and their conversion is explicitly unavailable") + w.add_argument("--site-bundle", action="append", default=None, metavar="PATH", + help="a site scene package whose site prefab colliders go into " + "collision.json; repeatable. Without it every scene package " + "under --bundle-root is used") + w.add_argument("--out-dir", required=True) + w.add_argument("--master", help="directory of caller-supplied master tables") + w.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + w.add_argument("--master-cache", help="where fetched tables are cached") + w.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it; " + "without it PATH is searched and the audio " + "entry says what is missing") + w.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + a = sub.add_parser("mysekai-audio", + help="extract the rest of the sound corpus: talk voices " + "named by talks.json, plus the remaining se, bgm " + "and part-voice packages") + a.add_argument("--talks", required=True, + help="a talk corpus in either shape this repo's extractors " + "write (talks grouped under units, or a flat talks " + "list); its voice cues are the talk-voice denominator, " + "and only those cues are decoded") + a.add_argument("--manifest", required=True, + help="asset bundle manifest giving the family denominators") + a.add_argument("--bundle-root", required=True, + help="directory holding the decrypted sound packages") + a.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "the products and whose loop.json is merged into") + a.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + a.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + ca = sub.add_parser("cutscene-audio", + help="decode the cues a cut-scene's own timeline names: its " + "voice clips from the cut-scene's birthday voice bank, " + "its music and effect clips from the packages they name") + ca.add_argument("--clips", required=True, action="append", + help="a cut-scene clip document (cutscene-timeline/clips/" + ".json); repeat for each cut-scene") + ca.add_argument("--cut-scenes", required=True, + help="the cut-scene master table (mysekaiCutScenes.json), " + "which routes voice clips to their voice bank") + ca.add_argument("--manifest", required=True, + help="asset bundle manifest the named packages are checked against") + ca.add_argument("--bundle-root", required=True, + help="directory holding the decrypted sound packages") + ca.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "the products and whose loop.json and corpus.json are merged into") + ca.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + ca.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + sq = sub.add_parser("audio-sequences", + help="attach the cues' sequence structure (type, repeat, " + "per-track delay, track -> stream) to an existing " + "loop sidecar, reading the archives again from the " + "decrypted tree; writes a copy, never in place") + sq.add_argument("--loop", required=True, help="the loop sidecar to read") + sq.add_argument("--bundle-root", required=True, action="append", + help="directory holding the decrypted sound packages; repeat " + "to give further trees, tried in order per entry") + sq.add_argument("--out", required=True, + help="where the annotated copy is written (must not be --loop)") + ra = sub.add_parser("record-audio", + help="read the waveform facts (length, loop region or none, " + "cue commands) of every package a music record can " + "play; the audio itself is not written") + ra.add_argument("--master", help="directory of caller-supplied master tables") + ra.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + ra.add_argument("--master-cache", help="where fetched tables are cached") + ra.add_argument("--bundle-root", required=True, action="append", + help="directory holding decrypted bundles; repeat to give further trees") + ra.add_argument("--bundle-version", required=True, + help="the asset version the bundles were downloaded at (written on every row)") + ra.add_argument("--only", choices=("music", "music_sound_track"), + help="read only the packages of this record track type") + ra.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + ra.add_argument("--out", required=True, help="the document to write") + rs = sub.add_parser("record-audio-storage", + help="add each row's storage mirror audio file: sha256 and size of " + "the downloaded bytes and the listing's version") + rs.add_argument("input", help="a record-audio document") + rs.add_argument("--server", required=True, choices=("jp", "cn")) + rs.add_argument("--workers", type=int, default=4) + rs.add_argument("--out", required=True) + rm = sub.add_parser("record-audio-merge", + help="combine record-audio documents of separate runs") + rm.add_argument("inputs", nargs="+") + rm.add_argument("--out", required=True) + rc = sub.add_parser("record-audio-carry", + help="carry a read record-audio document's facts to another " + "region's packages whose bundles have the same crc and size") + rc.add_argument("--master", help="directory of the target region's master tables") + rc.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + rc.add_argument("--master-cache", help="where fetched tables are cached") + rc.add_argument("--manifest", required=True, + help="the target region's package list (AssetBundleInfoNew)") + rc.add_argument("--source", required=True, + help="a record-audio document with storage attached") + rc.add_argument("--source-manifest", required=True, + help="the package list of the bundles the source document was read from") + rc.add_argument("--region", required=True, choices=("jp", "cn"), + help="the target region, written on the document") + rc.add_argument("--out", required=True) + rf = sub.add_parser("record-audio-fetch", + help="download and check (size, uncompressed crc) the bundles of " + "the packages a record-audio document lists as absent") + rf.add_argument("document", help="a record-audio document (its absent list is fetched)") + rf.add_argument("--manifest", required=True, + help="the package list (AssetBundleInfoNew) the bundles are fetched by") + rf.add_argument("--asset-base-url", required=True, + help="the prefix before each entry's downloadPath") + rf.add_argument("--workers", type=int, default=4) + rf.add_argument("--out", required=True, help="workspace; bundles land in /decrypted") + ac = sub.add_parser("audio-categories", + help="write the sound configuration's category table (name, " + "index, volume) the runtime scales cues by") + ac.add_argument("--configuration", required=True, + help="the sound configuration (.acf), or the application " + "package (zip) that holds assets/*.acf") + ac.add_argument("--out", required=True, + help="where categories.json is written (the runtime reads " + "phenomena/audio/categories.json)") + r = sub.add_parser("partvoice-routes", + help="build the speaker-to-partvoice-package routing " + "table from master tables and the roster") + r.add_argument("--master", help="directory of caller-supplied master tables") + r.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + r.add_argument("--master-cache", help="where fetched tables are cached") + r.add_argument("--roster", required=True, + help="characters.json, the registry artifact; its " + "characters are the participants' universe and order") + r.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "partvoice.json") + u = sub.add_parser("ui-action-icons", + help="export the standalone action-button icon bundles " + "as one PNG per bundle plus a manifest") + u.add_argument("--bundle", action="append", required=True, + help="an action-icon bundle; repeat for as many as wanted") + u.add_argument("--out-dir", required=True) + portraits = sub.add_parser("browser-assets", + help="export source character portraits for the browser catalog") + portraits.add_argument("--player-data", required=True, + help="the APK player data containing CharacterSDIconAtlas") + portraits.add_argument("--asset-root", required=True, + help="runtime resource root containing characters.json") + portraits.add_argument("--source-region", required=True, choices=("cn", "jp")) + portraits.add_argument("--source-game-version", required=True, + help="game version of the supplied UI player data") + presentation = sub.add_parser("fixture-presentation", + help="resolve model, wallpaper, floor and custom furniture source routes") + presentation.add_argument("--asset-root", required=True) + presentation.add_argument("--master", required=True) + presentation.add_argument("--bundles", required=True) + validate = sub.add_parser("validate-runtime", help="verify referenced resources and exact audio cue coverage") + validate.add_argument("--asset-root", required=True) + validate.add_argument("--verify-actor-hashes", action="store_true") + fixture_animations = sub.add_parser("fixture-animations", + help="refresh source-bound fixture clips and their authored time domain") + fixture_animations.add_argument("--asset-root", required=True) + fixture_animations.add_argument("--bundles", required=True) + fixture_animations.add_argument("--package", action="append") + fixture_animations.add_argument("--plan", action="store_true") + capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") + capabilities.add_argument("--asset-root", required=True) + lay = sub.add_parser("ui-layout", + help="extract the RectTransform hierarchy of the UI " + "prefabs (screen layers, dialogs) the player data " + "builds, plus a census of the UI package families") + lay.add_argument("--region", choices=("cn", "jp"), default="cn", + help="cn: one player data file whose resources.assets holds " + "every prefab; jp: the base APK and the asset pack, " + "whose Resources index points at one serialized file " + "per prefab") + lay.add_argument("--player-data", default=None, + help="cn: the APK player data file; every screen and dialog " + "prefab lives in its resources.assets (Resources.Load " + "fetches them, so no bundle name routes to them)") + lay.add_argument("--apk", action="append", default=None, + help="jp: an APK whose assets/bin/Data members are player " + "files; give the base APK and the asset pack") + lay.add_argument("--client-version", default=None, + help="jp: the client version the manifest and every document " + "name (ui/jp_decoders.py names the supported ones; each " + "object's chain comes from its serialized type hash); cn: " + "the client version the palette and tween defaults name") + lay.add_argument("--master", default=None, + help="jp: directory of master tables holding wordings.json") + lay.add_argument("--prefab", action="append", default=None, + help="a prefab name to extract (repeatable); a targeted " + "run writes those documents and omits the broad census") + lay.add_argument("--out-dir", required=True, + help="the output directory (ui-layout/) that receives one " + "document per prefab plus census.json") + lay.add_argument("--bundles", default=None, + help="optional decrypted-package directory; named so the " + "census also reports what the per-screen download " + "packages contain (textures and effects, not layout)") + lay.add_argument("--font-bundles", default=None, + help="jp: decrypted-package directory holding font/common " + "(jp download --root font/common); defaults to --bundles") + lay.add_argument("--bundle-manifest", default=None, + help="optional AssetBundleInfoNew.json; supplies the " + "isBuiltin split and declared sizes for the census") + words = sub.add_parser("ui-wordings", + help="write the runtime wording dictionary of a container build: " + "the built-in wording CSV of its language, then the master " + "wordings over it (the document a shared root reads as its " + "wordings.json)") + words.add_argument("--player-data", required=True, + help="the APK player data file whose Resources index holds the wording CSV") + words.add_argument("--master", required=True, + help="directory of master tables holding wordings.json") + words.add_argument("--client-version", required=True, + help="the client version the document's source block names") + words.add_argument("--out", required=True, help="the document path") + reactions = sub.add_parser("fixture-reactions", + help="export the native furniture reaction-character groups") + reactions.add_argument("--bundle", required=True, + help="decrypted mysekai__system__fixture_reaction_data bundle") + reactions.add_argument("--out", required=True, + help="output fixture-reactions.json path") + bem = sub.add_parser("birthday-end-motions", + help="export the waving motion the birthday party " + "plays as it closes (row per character unit)") + bem.add_argument("--bundle", required=True, + help="decrypted mysekai__birthday__birthday_end_character_motion_data_list bundle") + bem.add_argument("--out", required=True, + help="output birthday-end-character-motions.json path") + bpt = sub.add_parser("birthday-party-talks", + help="attach the birthday-party scripts one master " + "table names to an existing talks.json") + bpt.add_argument("--master", help="directory of caller-supplied master tables") + bpt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bpt.add_argument("--master-cache", help="where fetched tables are cached") + bpt.add_argument("--bundle", required=True, help="talk scenario bundle") + bpt.add_argument("--lib", help="decrypted lib package carrying the constant " + "tables the scripts name") + bpt.add_argument("--talks", required=True, + help="an existing talks.json whose birthdayParty block is " + "computed from this master partition") + bpt.add_argument("--out", required=True) + f = sub.add_parser("fixture-master-slice", + help="slice mysekaiFixtures down to the columns the " + "runtime reads (button gating and footprint)") + f.add_argument("--master", help="directory of caller-supplied master tables") + f.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + f.add_argument("--master-cache", help="where fetched tables are cached") + f.add_argument("--out", required=True) + q = sub.add_parser("site", help="extract the site (place) asset packages") + q.add_argument("--bundle", action="append", required=True, + help="a package under the site path; repeat for as many as wanted") + q.add_argument("--bundle-root", + help="directory the packages a bundle depends on are read from") + q.add_argument("--out-dir", required=True) + q.add_argument("--master", help="directory of caller-supplied master tables") + q.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + q.add_argument("--master-cache", help="where fetched tables are cached") + ua = sub.add_parser("ui-audio", help="extract embedded UI cue sheets with the shared audio decoder") + ua.add_argument("--apk", required=True) + ua.add_argument("--out", required=True) + ua.add_argument("--vgmstream") + ua.add_argument("--ffmpeg") + sls = sub.add_parser("sitemap-screen-layer", + help="write site/sitemap/screen_layer/screen_layer.json from APK " + "members (player data without resources.assets)") + sls.add_argument("--apk", action="append", required=True, + help="an APK whose assets/bin/Data members are player files; " + "give the base APK and the asset pack") + sls.add_argument("--client-version", required=True, + help="the client version the field chains are for") + sls.add_argument("--out-dir", required=True, + help="runtime root; the document lands at " + "site/sitemap/screen_layer/screen_layer.json under it") + mth = sub.add_parser("material-thumbnails", + help="write material-thumbnails.json and textures/material{id}.png " + "from the thumbnail/material package, one per materials row") + mth.add_argument("--bundle", required=True, help="the decrypted thumbnail/material package") + mth.add_argument("--master", required=True, + help="directory of master tables holding materials.json") + mth.add_argument("--out-dir", required=True, + help="output directory (material-thumbnails/ of a runtime root)") + hon = sub.add_parser("honor-images", + help="write honor/index.json and the textures of the honor bundles " + "the birthday-party delivery rewards' honors draw") + hon.add_argument("--master", required=True, + help="directory of master tables (the delivery total rewards, resource boxes, honors, honor groups)") + hon.add_argument("--bundles", required=True, help="directory of decrypted honor and honor_frame packages") + hon.add_argument("--out-dir", required=True, help="runtime root; images land under honor/ and honor_frame/") + ipv = sub.add_parser("item-icons", + help="write index.json and textures///.png from " + "the mysekai item_preview and thumbnail material/item/tool packages, " + "keyed by the package path") + ipv.add_argument("--decrypted", required=True, + help="directory holding the decrypted mysekai item icon packages") + ipv.add_argument("--root", default=None, + help="runtime root whose mysekai-materials/items/fixtures/tools documents " + "give the coverage report") + ipv.add_argument("--out-dir", required=True, + help="output directory (mysekai-item-icons/ of a runtime root)") + a = sub.add_parser("fetch-apk", help="discover, download, or inspect an Android APK") + a.add_argument("--endpoint", default=None); a.add_argument("--timeout", type=float, default=30.0); a.add_argument("--retries", type=int, default=3) + asub = a.add_subparsers(dest="apk_command", required=True) + asub.add_parser("latest") + ad = asub.add_parser("download"); ad.add_argument("destination"); ad.add_argument("--url"); ad.add_argument("--sha256") + ai = asub.add_parser("inspect"); ai.add_argument("apk") + rp = sub.add_parser( + "player-render-pipeline", + help="extract the player-shipped MYSEKAI renderer pipeline and feature settings", + ) + rp.add_argument( + "--player-data", required=True, + help="extracted assets/bin/Data root containing globalgamemanagers", + ) + rp.add_argument( + "--types", required=True, + help="decoded managed assemblies for the same player build", + ) + rp.add_argument( + "--asset-pack", required=True, + help="APK/zip containing serialized resource files referenced by ResourceManager", + ) + rp.add_argument("--game-version", help="caller-known game version recorded as provenance") + rp.add_argument("--out", required=True, help="output JSON evidence path") + fal = sub.add_parser("fixture-actors", help="export all source-qualified character furniture animations") + fal.add_argument("--assets", required=True); fal.add_argument("--master", required=True) + fal.add_argument("--bundles", required=True); fal.add_argument("--workers", type=int, default=2) + fal.add_argument("--plan", action="store_true", help="write and summarize the source dependency plan without exporting") + fal.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="rotation key storage (default: snorm16); float32 disables lossy encoding") + fal.add_argument("--relation", action="append", choices=("fixture", "cut_scene"), default=None, + help="export only these relations' libraries (repeatable; default: all)") + ptc = sub.add_parser("player-timeline-clips", + help="export each player fixture timeline's CharacterAnimator clips against the " + "player avatar rig, one clip file per timeline package") + ptc.add_argument("--assets", required=True, + help="asset root holding fixture-timeline/ and the player timeline rows") + ptc.add_argument("--bundles", required=True, + help="directory of decrypted bundles (the avatar model and the fixture models)") + ptc.add_argument("--plan", action="store_true", help="summarize the clip plan without exporting") + stl = sub.add_parser("site-timelines", + help="export a site package's timeline tables, its step items (each prefab its " + "own glb with its prop clips) and its scene directors' rigs") + stl.add_argument("--assets", required=True, help="asset root receiving site-timeline/") + stl.add_argument("--bundles", required=True, + help="directory of decrypted bundles (the site packages and their dependencies)") + stl.add_argument("--package", action="append", required=True, + help="site package to export, e.g. mysekai__site__field__festivalgarden; repeatable") + fc = sub.add_parser("fixture-coverage", help="join every fixture master ID to extracted coverage") + fc.add_argument("--data-root", default=DEFAULT_DATA_ROOT); fc.add_argument("--out") + fc.add_argument("--region"); fc.add_argument("--source-version"); fc.add_argument("--json", action="store_true") + jp = sub.add_parser("jp", help="discover and download Japanese-server original bundles") + jsub = jp.add_subparsers(dest="jp_command", required=True) + jd = jsub.add_parser("discover", help="compare bootstrap metadata with the live official version list") + jd.add_argument("--bootstrap-url", default=None); jd.add_argument("--timeout", type=float, default=30.0) + jd.add_argument("--retries", type=int, default=3) + jm = jsub.add_parser("manifest", help="create a versioned JP bundle descriptor") + jm.add_argument("--app-version"); jm.add_argument("--app-hash") + jm.add_argument("--asset-version"); jm.add_argument("--asset-hash") + jm.add_argument("--versions-url", default=None) + jm.add_argument("--guest", action="store_true", help="explicitly create a guest only when public asset identity is unavailable") + jm.add_argument("--session", help="private guest-session JSON path; required with --guest") + jm.add_argument("--out", required=True); jm.add_argument("--timeout", type=float, default=30.0) + jm.add_argument("--retries", type=int, default=3) + jm.add_argument("--json", action="store_true", help="print the full descriptor as well as writing --out") + jpl = jsub.add_parser("plan", help="dependency-closed dry run from an online or offline descriptor") + jpl.add_argument("--manifest", required=True); jpl.add_argument("--family", action="append", default=[]) + jpl.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") + jpl.add_argument("--json", action="store_true", help="print root and bundle lists, not only counts") + jrt = jsub.add_parser("prepare", help="assemble a region-qualified Moly runtime in a separate staging directory") + jrt.add_argument("--manifest", required=True); jrt.add_argument("--workspace", required=True) + jrt.add_argument("--out", required=True); jrt.add_argument("--ui-from", required=True) + jrt.add_argument("--ui-jp-from", required=True, + help="JP UI extraction (ui-layout --region jp); its documents replace their donor copies") + jrt.add_argument("--vgmstream"); jrt.add_argument("--ffmpeg") + jrt.add_argument("--master-from", help="completed source-pinned master directory; defaults to Haruki when available") + jrt.add_argument("--actor-workers", type=int, default=2) + jrt.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") + jms = jsub.add_parser("master", help="download an immutable, bundle-matched JP master snapshot") + jms.add_argument("--manifest", required=True); jms.add_argument("--out", required=True) + jms.add_argument("--revision", default="main"); jms.add_argument("--workers", type=int, default=6) + jms.add_argument("--provider", choices=("haruki", "sekai-world"), default="haruki") + jms.add_argument("--all-tables", action="store_true", help="fetch all master tables, not only Moly dependencies") + jdl = jsub.add_parser("download", help="download and verify original JP bundles without publishing an extracted release") + jdl.add_argument("--manifest", required=True); jdl.add_argument("--out") + jdl.add_argument("--workers", type=int, default=8); jdl.add_argument("--retries", type=int, default=4) + jdl.add_argument("--root", action="append", default=[]) + jdl.add_argument("--family", action="append", default=[]) + jpu = jsub.add_parser("pull", help="download, decrypt and extract a JP descriptor") + jpu.add_argument("--manifest", required=True); jpu.add_argument("--out") + jpu.add_argument("--extract-out", default="local-data-jp") + jpu.add_argument("--workers", type=int, default=4); jpu.add_argument("--retries", type=int, default=4) + jpu.add_argument("--master"); jpu.add_argument("--master-cache") + jpu.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") + s = sub.add_parser("shader", help="what a package's shaders declare and contain") + s.add_argument("package", help="one AssetBundle file") + s.add_argument("--platform", type=int, default=None, + help="count only this ShaderCompilerPlatform (9 is GLSL text)") + s.add_argument("--out", help="write the full census here as JSON") + s.add_argument("--programs", metavar="DIR", + help="write each program body here, named by shader, platform and record") + s.add_argument("--json", action="store_true", help="print the counts as JSON") + + args = ap.parse_args(argv); _configure_unity(args.unity_version) + if args.cmd == "fixture-actors": + from chara.fixture_actor_libraries import build_plan, export_fixture_actor_libraries + if args.plan: + report = build_plan(args.assets, args.master, args.bundles, rotation_encoding=args.rotation_encoding) + print(json.dumps({"summary": report["summary"], "gaps": report["gaps"]}, ensure_ascii=False)) + return 1 if report["gaps"] else 0 + report = export_fixture_actor_libraries(args.assets, args.master, args.bundles, + workers=args.workers, unity_version=args.unity_version or "2022.3.62f3", + rotation_encoding=args.rotation_encoding, relations=args.relation, + progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) + print(json.dumps({key: value for key, value in report.items() if key != "results"}, ensure_ascii=False)) + return 1 if report["failed"] else 0 + if args.cmd == "player-timeline-clips": + from chara.player_timeline_clips import build_plan, export_player_timeline_clips + if args.plan: + plan = build_plan(args.assets) + print(json.dumps({"summary": plan["summary"], "gaps": plan["gaps"]}, ensure_ascii=False)) + return 1 if plan["gaps"] else 0 + report = export_player_timeline_clips(args.assets, args.bundles, + progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) + print(json.dumps({key: value for key, value in report.items() if key != "packages"}, + ensure_ascii=False)) + return 1 if report["gaps"] else 0 + if args.cmd == "site-timelines": + from sites.step_items import export_site_timelines + report = export_site_timelines(args.bundles, args.assets, list(args.package)) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "fixture-coverage": + from fixtures.coverage import write_coverage + path, report = write_coverage(args.data_root, args.out, region=args.region, + source_version=args.source_version) + print(json.dumps(report if args.json else {**report["summary"], "report": str(path)}, ensure_ascii=False)) + return 1 if report["summary"]["runtime"]["failed"] else 0 + if args.cmd == "jp": + from core import jp as jp_assets + try: + if args.jp_command == "discover": + options = {"timeout": args.timeout, "retries": args.retries} + if args.bootstrap_url: options["bootstrap_url"] = args.bootstrap_url + report = jp_assets.discover(**options) + elif args.jp_command == "manifest": + report = jp_assets.fetch_manifest(args.out, app_version=args.app_version, + app_hash=args.app_hash, session_file=args.session, + asset_version=args.asset_version, + asset_hash=args.asset_hash, guest=args.guest, + **({"versions_url": args.versions_url} + if args.versions_url else {}), + timeout=args.timeout, retries=args.retries) + elif args.jp_command == "plan": + report = jp_assets.plan(args.manifest, families=args.family, include=args.root) + elif args.jp_command == "prepare": + from core.runtime_pack import prepare_runtime + report = prepare_runtime(args.manifest, args.workspace, args.out, ui_from=args.ui_from, + ui_jp_from=args.ui_jp_from, + decoder=args.vgmstream, transcoder=args.ffmpeg, + master_from=args.master_from, actor_workers=args.actor_workers, + rotation_encoding=args.rotation_encoding, + progress=lambda label: print(label, file=sys.stderr)) + elif args.jp_command == "master": + from core.jp_master import download_master_snapshot + report = download_master_snapshot(args.manifest, args.out, revision=args.revision, workers=args.workers, provider=args.provider, all_tables=args.all_tables) + report = {"identity": report["identity"], "tableCount": report["tableCount"], "out": args.out} + elif args.jp_command == "download": + from core.jp_download import download_descriptor + report = download_descriptor(args.manifest, out=args.out, workers=args.workers, + retries=args.retries, include=args.root, families=args.family, + progress=lambda row: print(f"{row['status']}: {row['completed']}/{row['requested']} verified, {row['failed']} failed", file=sys.stderr)) + else: + report = jp_assets.pull_descriptor(args.manifest, out=args.out, + extract_out=args.extract_out, workers=args.workers, + retries=args.retries, master=args.master, + master_cache=args.master_cache, + include=args.root) + if args.jp_command == "manifest" and not args.json: + shown = {"source": report["source"], "bundles": len(report["bundles"]), "descriptor": args.out} + elif args.jp_command == "plan" and not args.json: + shown = {"source": report["source"], **report["summary"]} + else: + shown = report + print(json.dumps(shown, ensure_ascii=False)); return 0 + except (jp_assets.JPError, OSError, ValueError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr); return 1 + if args.cmd == "pull": + from core.fetch import pull + source, cache = _master_source(args) + report = pull(args.manifest, args.out, args.asset_base_url, args.workers, args.retries, + master=source, master_cache=cache, extract_out=args.extract_out, + vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) + if args.json: + print(json.dumps(report, ensure_ascii=False)) + else: + print(f"downloaded {report['downloads']}/{report['requiredBundles']} bundles") + # The sound packages are the one part of the set that master rows name, + # so "no audio" has a reason the caller must see without asking for JSON. + audio = report.get("audio") or {} + if audio.get("roots"): + print(f"audio: {len(audio['roots'])} sound packages named by master tables") + elif audio.get("error"): + print(f"audio: no sound package pulled ({audio['error']})") + if audio.get("notInManifest"): + print(f"audio: {len(audio['notInManifest'])} named sound packages " + f"are not in this manifest: " + f"{', '.join(audio['notInManifest'])}") + _print_extract_summary(report["extraction"], args.extract_out or DEFAULT_DATA_ROOT) + return 1 if _extract_failed(report["extraction"]) else 0 + if args.cmd == "shader": + import UnityPy + from shaders import census as shader_census + entries = shader_census.census(UnityPy.load(args.package), + keep_code=bool(args.programs)) + counts = shader_census.totals(entries, args.platform) + if args.out: + from core.jsonio import write_json + write_json(args.out, {"package": args.package, + "shaders": shader_census.without_code(entries)}) + if args.programs: + written = _write_programs(args.programs, entries, args.platform) + counts["bodiesWritten"] = written + if args.json: + print(json.dumps(counts, ensure_ascii=False)) + else: + scope = "all platforms" if args.platform is None else f"platform {args.platform}" + print(f"{counts['shaders']} shaders, {counts['records']} program records, " + f"{counts['uniquePrograms']} distinct programs ({scope})") + if counts["platformErrors"]: + print(f"{counts['platformErrors']} platform blob(s) failed to parse") + for name, records, unique in shader_census.by_shader(entries, args.platform)[:10]: + print(f" {unique:6d} distinct / {records:6d} records {name}") + return 1 if counts["platformErrors"] else 0 + if args.cmd == "ui-audio": + from ui.audio import extract_ui_audio + report = extract_ui_audio(args.apk, args.out, decoder=args.vgmstream, transcoder=args.ffmpeg) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "fetch-apk": + from core import apk + forwarded = [] + if args.endpoint: forwarded += ["--endpoint", args.endpoint] + forwarded += ["--timeout", str(args.timeout), "--retries", str(args.retries), args.apk_command] + if args.apk_command == "download": + forwarded += [args.destination] + if args.url: forwarded += ["--url", args.url] + if args.sha256: forwarded += ["--sha256", args.sha256] + elif args.apk_command == "inspect": + forwarded += [args.apk] + return apk.main(forwarded) + if args.cmd == "player-render-pipeline": + from core.player_render_pipeline import ( + extract_mysekai_render_pipeline, + write_mysekai_render_pipeline, + ) + try: + document = extract_mysekai_render_pipeline( + args.player_data, + args.types, + args.asset_pack, + unity_version=args.unity_version, + game_version=args.game_version, + ) + write_mysekai_render_pipeline(args.out, document) + except (OSError, ValueError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + first = document["rendererSlots"][0]["objectKey"] if document["rendererSlots"] else None + count = len(document["renderers"].get(first, {}).get("features", [])) if first else 0 + print(json.dumps({ + "unityVersion": document["unityVersion"], + "gameVersion": document["gameVersion"], + "rendererSlots": len(document["rendererSlots"]), + "primaryRendererFeatures": count, + "out": args.out, + }, ensure_ascii=False)) + return 0 + if args.cmd == "extract": + from core.extract import extract_manifest + from core.assets.packages import builtin_archive_paths + source, cache = _master_source(args) + report = extract_manifest(args.manifest, args.bundles, args.out, args.unity_version, + master=source, master_cache=cache, + player_data=args.player_data, + player_types=args.player_types, + player_declarations=args.player_declarations, + fixture_meshes=args.fixture_meshes, + fixture_particles=args.fixture_particles, + rotation_encoding=args.rotation_encoding, + shader_compiler=args.shader_compiler, + builtin_resources=builtin_archive_paths( + args.builtin_resources), + vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) + if args.json: + print(json.dumps(report, ensure_ascii=False)) + else: + if report.get("discovery"): + d = report["discovery"] + print(f"discovered {d['selected']} pack bundles under {args.bundles} " + f"(ignored {d['ignored']} unrelated files)") + _print_extract_summary(report, args.out) + return 1 if _extract_failed(report) else 0 + from chara import characters + if args.cmd == "motion-library": + from chara.motion_library import export_motion_library + names = [s for s in args.clips.split(",") if s] if args.clips else None + report = export_motion_library(args.reference_bundle, args.motion_bundle, args.out_dir, args.name, + aux=tuple(args.aux_bundle), names=names, + rotation_encoding=args.rotation_encoding, + controller_bundles=args.controller_bundle) + print(json.dumps(report, ensure_ascii=False)); return 0 + if args.cmd == "registry": + source, cache = _master_source(args) + if not source: + ap.error("registry needs master tables: pass --master or --master-url") + from core.assets.manifest import parse_manifest + from core.assets.router import route + from core.extract import _unit_id + from core.master import Master, MissingTable + from chara.registry import build_registry + names = parse_manifest(args.manifest) + units = sorted({u for n in names if route(n) and route(n).domain == "character" + for u in [_unit_id(n)] if u is not None}) + index = None + if args.motion_index: + with open(args.motion_index, encoding="utf-8") as handle: + index = json.load(handle) + source, cache = _master_source(args) + master = Master(source, cache_dir=cache) + try: + client_configs = master.client_configs() + except MissingTable: + client_configs = None + document = build_registry(master, units, index, client_configs=client_configs) + with open(args.out, "w", encoding="utf-8", newline="\n") as handle: + json.dump(document, handle, ensure_ascii=False, indent=1, allow_nan=False) + handle.write("\n") + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + + if args.cmd == "official-text": + from chara.text_catalog import match_official + from core.jsonio import write_json + with open(args.source, encoding="utf-8") as handle: + source = json.load(handle) + with open(args.donor, encoding="utf-8") as handle: + donor = json.load(handle) + result = match_official(source, donor, locale=args.locale) + write_json(args.out, result) + print(json.dumps({"entries": len(result["entries"]), "issues": len(result["issues"]), + "revision": result["revision"]})) + return 0 + + if args.cmd == "text-catalog": + from chara.text_catalog import export_catalog + print(json.dumps(export_catalog(args.corpus, args.out, region=args.region, + source_revision=args.source_revision, + previous=args.previous), ensure_ascii=False)) + return 0 + + if args.cmd == "talks": + source, cache = _master_source(args) + if not source: + ap.error("talks needs master tables: pass --master or --master-url") + from chara.talks import extract_talks + source, cache = _master_source(args) + print(json.dumps(extract_talks(source, args.bundle, args.out, + master_cache=cache, + lib_bundle=args.lib), ensure_ascii=False)) + return 0 + + if args.cmd == "tweets": + source, cache = _master_source(args) + if not source: + ap.error("tweets needs master tables: pass --master or --master-url") + from chara.tweets import extract_tweets + print(json.dumps(extract_tweets(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "tweet-tables": + source, cache = _master_source(args) + if not source: + ap.error("tweet-tables needs master tables: " + "pass --master or --master-url") + from chara.tweet_tables import extract_tweet_tables + print(json.dumps(extract_tweet_tables(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "site-groups": + source, cache = _master_source(args) + if not source: + ap.error("site-groups needs master tables: " + "pass --master or --master-url") + from sites.site_groups import extract_site_groups + print(json.dumps(extract_site_groups(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "ui-action-icons": + from ui.action_icon import export_action_icons + print(json.dumps(export_action_icons(args.bundle, args.out_dir), + ensure_ascii=False)) + return 0 + + if args.cmd == "browser-assets": + from ui.character_portrait import export_character_portraits + result = export_character_portraits(args.player_data, args.asset_root, + source_region=args.source_region, + source_game_version=args.source_game_version) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 + + if args.cmd == "fixture-presentation": + from fixtures.presentation import extract_custom_geometry, enrich_presentation + extracted = extract_custom_geometry(args.bundles, Path(args.asset_root) / "custom-fixture-models") + if extracted.get("failed"): + print(json.dumps({"customGeometry": extracted}, ensure_ascii=False)) + return 1 + print(json.dumps(enrich_presentation(args.master, args.asset_root), ensure_ascii=False)) + return 0 + + if args.cmd == "validate-runtime": + from core.runtime_validation import validate_runtime + report = validate_runtime(args.asset_root, verify_actor_hashes=args.verify_actor_hashes) + print(json.dumps(report, ensure_ascii=False)) + return 1 if report["missingResources"] else 0 + + if args.cmd == "fixture-animations": + from fixtures.animation_refresh import plan_animation_refresh, refresh_fixture_animations + report = plan_animation_refresh(args.asset_root) if args.plan else refresh_fixture_animations( + args.asset_root, args.bundles, packages=args.package, + progress=lambda row: print(json.dumps(row), file=sys.stderr)) + print(json.dumps(report, ensure_ascii=False)) + return 0 + + if args.cmd == "timeline-capabilities": + from fixtures.timeline_capabilities import build_capabilities + report = build_capabilities(args.asset_root) + print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) + return 0 + + if args.cmd == "ui-layout" and args.region == "jp": + from ui.prefab_layout import extract_layout_jp + if not (args.apk and args.client_version and args.master and args.prefab): + ap.error("ui-layout --region jp needs --apk (base and asset pack), " + "--client-version, --master and at least one --prefab") + result = extract_layout_jp(args.apk, args.out_dir, client_version=args.client_version, + master_dir=args.master, prefabs=args.prefab, + bundles_root=args.bundles, + font_bundles_root=args.font_bundles) + for item in result["written"]: + summary = item["summary"] + print(f"layout: {item['family']}/{item['prefab']} " + f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " + f"texts={summary['textComponents']} " + f"partial={len(summary['partialComponents'])} " + f"unresolved={len(summary['unresolvedPointers'])}") + for item in result["failures"]: + print(f"layout FAILED {item['prefab']}: {item['reason']}") + print(f"manifest: {result['manifestPath']}") + return 0 if not result["failures"] else 1 + + if args.cmd == "ui-layout": + from ui.prefab_layout import extract_layout + if not args.player_data: + ap.error("ui-layout --region cn needs --player-data") + result = extract_layout(args.player_data, args.out_dir, + bundles_root=args.bundles, + bundle_manifest=args.bundle_manifest, + prefabs=args.prefab, + client_version=args.client_version) + for item in result["settings"]: + print(f"settings: {item['document']} from {item['serializedFile']}:{item['pathId']}") + if not result["settings"]: + print("settings: palette.json and tween-settings.json not written " + "(they name the client version: pass --client-version)") + for item in result["written"]: + summary = item["summary"] + print(f"layout: {item['family']}/{item['prefab']} " + f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " + f"texts={summary['textComponents']} " + f"partial={len(summary['partialComponents'])}") + for item in result["failures"]: + print(f"layout FAILED {item['prefab']}: {item['reason']}") + print(f"census: {result['censusPath']}") + return 0 if not result["failures"] else 1 + + if args.cmd == "ui-wordings": + from ui.prefab_layout import extract_container_wordings + result = extract_container_wordings(args.player_data, args.master, args.out, args.client_version) + print(f"wordings: {result['rows']} entries ({result['builtinRows']} built-in, " + f"{result['masterRows']} master rows, {result['masterOverwroteBuiltin']} built-in " + f"overwritten, {result['masterValuesRewritten']} master values rewritten, " + f"{result['builtinRejectedLines']} rejected lines) from " + f"{result['serializedFile']}:{result['pathId']} -> {result['path']}") + return 0 + + if args.cmd == "fixture-reactions": + from fixtures.reactions import extract_fixture_reactions + print(json.dumps(extract_fixture_reactions(args.bundle, args.out), ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-end-motions": + from sites.birthday_end_motions import extract_birthday_end_motions + print(json.dumps(extract_birthday_end_motions(args.bundle, args.out), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-party-talks": + source, cache = _master_source(args) + if not source: + ap.error("birthday-party-talks needs master tables: " + "pass --master or --master-url") + from chara.talks import attach_birthday_talks + print(json.dumps(attach_birthday_talks(source, args.bundle, args.talks, + args.out, master_cache=cache, + lib_bundle=args.lib), + ensure_ascii=False)) + return 0 + + if args.cmd == "fixture-master-slice": + source, cache = _master_source(args) + if not source: + ap.error("fixture-master-slice needs master tables: " + "pass --master or --master-url") + from fixtures.master_slice import export_fixture_master_slice + print(json.dumps(export_fixture_master_slice(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-parties": + source, cache = _master_source(args) + if not source: + ap.error("birthday-parties needs master tables: " + "pass --master or --master-url") + from sites.birthday_parties import extract_birthday_parties + print(json.dumps(extract_birthday_parties(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-party-delivery": + source, cache = _master_source(args) + if not source: + ap.error("birthday-party-delivery needs master tables: " + "pass --master or --master-url") + from sites.birthday_party_delivery import extract_birthday_party_delivery + print(json.dumps(extract_birthday_party_delivery(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "gate-visit-policy": + source, cache = _master_source(args) + if not source: + ap.error("gate-visit-policy needs master tables: " + "pass --master or --master-url") + from sites.gate_visit_policy import extract_gate_visit_policy + print(json.dumps(extract_gate_visit_policy(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + if args.cmd == "time-manager": + from core.time_manager import extract_time_manager + document = extract_time_manager(args.source, args.out) + print(json.dumps({"source": document["source"], "timeManager": document["timeManager"]}, + ensure_ascii=False)) + return 0 + if args.cmd == "client-config": + source, cache = _master_source(args) + if not source: + ap.error("client-config needs master tables: " + "pass --master or --master-url") + from core.client_config import extract_client_config + print(json.dumps(extract_client_config(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "master-tables": + source, cache = _master_source(args) + if not source: + ap.error("master-tables needs master tables: " + "pass --master or --master-url") + from core.master_tables import extract_master_tables + print(json.dumps(extract_master_tables(source, args.out, + master_cache=cache, tables=args.tables), + ensure_ascii=False)) + return 0 + + if args.cmd == "emoticons": + from chara.emoticons import extract_emoticons + print(json.dumps(extract_emoticons(args.bundle, args.out_dir), ensure_ascii=False)) + return 0 + if args.cmd == "avatar-parts": + from chara.avatar_parts import extract_avatar_parts + print(json.dumps(extract_avatar_parts(args.bundle, args.out_dir), ensure_ascii=False)) + return 0 + if args.cmd == "harvest-tools": + from chara.harvest_tools import extract_harvest_tools + result = extract_harvest_tools(args.bundle, args.out_dir, bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "player-tool-models": + from chara.harvest_tools import extract_player_tool_models + result = extract_player_tool_models(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "effect-resources": + from core.resource_exports import export_effect_resources + print(json.dumps(export_effect_resources( + args.player_data, args.types, args.out, declarations=args.declarations, + bundle_root=args.bundle_root), ensure_ascii=False)) + return 0 + if args.cmd == "layout-grid": + from core.layout_grid import export_layout_grid + print(json.dumps(export_layout_grid(args.player_data, args.types, args.out_dir), + ensure_ascii=False, indent=2)) + return 0 + if args.cmd == "drop-item-shade": + from core.resource_exports import export_drop_item_shade + print(json.dumps(export_drop_item_shade(args.player_data, args.types, args.out), + ensure_ascii=False)) + return 0 + if args.cmd == "stencil-shadow": + from core.resource_exports import export_stencil_shadow + print(json.dumps(export_stencil_shadow(args.player_data, args.types, args.out), + ensure_ascii=False)) + return 0 + if args.cmd == "resource-prefab": + from core.resource_exports import export_resource_prefab + from core.assets.packages import builtin_archive_paths + document = export_resource_prefab(args.player_data, args.types, + args.load_name, args.out_dir, + builtin_archive_paths(args.builtin_resources)) + print(json.dumps({"file": document["file"], "objects": document["objects"], + "roots": [root["name"] for root in document["roots"]], + "particles": len(document["particles"]), + "unsupported": len(document["unsupported"])}, + ensure_ascii=False)) + return 0 + if args.cmd == "resource-programs": + from core.resource_exports import export_resource_programs + document = export_resource_programs(args.player_data, args.types, args.load_name, + args.out_dir, args.shader_compiler) + print(json.dumps({"file": document["file"], + "shaders": [(row["name"], row["state"], len(row["materials"])) + for row in document["shaders"]]}, ensure_ascii=False)) + return 0 + if args.cmd == "fixture-particles": + from core.assets.packages import PackageStore, builtin_archive_paths, require_paths + from fixtures.particles import DOMAINS, extract_from_store + require_paths(args.bundle) + builtins = builtin_archive_paths(args.builtin_resources) + require_paths(builtins, "built-in resource") + store = PackageStore(list(args.bundle) + builtins, args.bundle_root) + result = extract_from_store(store, args.out_dir, + shader_compiler=args.shader_compiler, + domains=tuple(args.domain or DOMAINS)) + print(json.dumps(result, ensure_ascii=False)) + return 0 + if args.cmd == "prefab-package": + from sites.prefab_package import extract_prefab_package + print(json.dumps(extract_prefab_package(args.bundle, args.prefab, args.out_dir, + bundle_root=args.bundle_root), + ensure_ascii=False)) + return 0 + if args.cmd == "site-sound-wave": + from sites.sound_wave import extract_site_sound_wave + result = extract_site_sound_wave(args.bundle, args.out, bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if result["failed"] else 0 + if args.cmd == "site-common-action-effects": + from sites.action_effects import extract_site_common_action_effects + result = extract_site_common_action_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-harvest-object-effects": + from sites.action_effects import extract_site_harvest_object_effects + result = extract_site_harvest_object_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-action-effects": + from sites.action_effects import extract_site_action_effects + result = extract_site_action_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-home-effects": + from sites.action_effects import extract_site_home_effects + result = extract_site_home_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "fixture-put-effects": + from sites.action_effects import extract_fixture_put_effects + result = extract_fixture_put_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "player-avatar": + from chara.player_avatar import export_player_avatar, write_motion_manifest + record = export_player_avatar(args.bundle, args.out_dir, name=args.name) + manifest_path = str(Path(args.out_dir) / f"{args.name}.motion-manifest.json") + write_motion_manifest(record, manifest_path) + print(json.dumps({ + "glb": record["glb"], "index": record["index"], + "motionManifest": manifest_path, + "sourcePackages": record["sourcePackages"], + "playerMesh": record["playerMesh"], + "counts": record["counts"]}, ensure_ascii=False)) + return 0 + if args.cmd == "phenomena": + from core.assets.packages import builtin_archive_paths, require_paths + from phenomena.environments import extract_phenomena + source, cache = _master_source(args) + # --bundle takes paths, not bare package names: PackageStore keys the + # store by basename and UnityPy.load answers a path that is not there + # with an empty environment and no error, so a bare name runs green + # over zero objects. Refuse it here, before any work is done. + require_paths(args.bundle) + require_paths(args.site_bundle) + report = extract_phenomena(args.bundle, args.out_dir, + bundle_root=args.bundle_root, master=source, + master_cache=cache, vgmstream=args.vgmstream, + ffmpeg=args.ffmpeg, + shader_compiler=args.shader_compiler, + extra_archives=builtin_archive_paths( + args.builtin_resources), + site_bundles=args.site_bundle) + print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, + ensure_ascii=False)) + return 0 + if args.cmd == "mysekai-audio": + from phenomena.audio_corpus import extract_audio_corpus + report = extract_audio_corpus(args.talks, args.manifest, + args.bundle_root, args.out_dir, + decoder=args.vgmstream, + transcoder=args.ffmpeg) + print(json.dumps({k: v for k, v in report.items() if k != "audio"}, + ensure_ascii=False)) + return 0 + if args.cmd == "cutscene-audio": + from phenomena.cutscene_audio import extract_cutscene_audio + report = extract_cutscene_audio(args.clips, args.cut_scenes, args.manifest, + args.bundle_root, args.out_dir, + decoder=args.vgmstream, transcoder=args.ffmpeg) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "audio-sequences": + if Path(args.out).resolve() == Path(args.loop).resolve(): + ap.error("audio-sequences writes a copy: --out must differ from --loop") + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.audio_sequences import annotate_loop + report = annotate_loop(args.loop, args.bundle_root, args.out) + print(json.dumps({"counts": report["counts"], + "structuredByType": report["structuredByType"], + "archiveCuesByReferenceType": report["archiveCuesByReferenceType"], + "archiveSequencesByType": report["archiveSequencesByType"], + "answeredByTree": {k: len(v) for k, v + in report["answeredByTree"].items()}, + "failures": len(report["failures"]), + "disagreements": len(report["disagreements"]), + "sampleDifferences": len(report["sampleDifferences"])}, + ensure_ascii=False)) + return 0 + if args.cmd == "record-audio": + source, cache = _master_source(args) + if not source: + ap.error("record-audio needs master tables: " + "pass --master or --master-url") + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.record_audio import extract_record_audio + document = extract_record_audio(source, args.bundle_root, args.out, + bundle_version=args.bundle_version, + decoder=args.vgmstream, master_cache=cache, + only=args.only) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-storage": + from phenomena.record_audio import attach_storage + document = attach_storage(args.input, args.out, server=args.server, + workers=args.workers) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-carry": + source, cache = _master_source(args) + if not source: + ap.error("record-audio-carry needs the target master tables: " + "pass --master or --master-url") + from phenomena.record_audio import carry_record_audio + document = carry_record_audio(source, args.manifest, args.source, + args.source_manifest, args.out, + region=args.region, master_cache=cache) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-fetch": + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.record_audio import fetch_record_bundles + receipt = fetch_record_bundles(args.manifest, args.document, args.asset_base_url, + args.out, workers=args.workers) + print(json.dumps(receipt["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-merge": + from phenomena.record_audio import merge_record_audio + print(json.dumps(merge_record_audio(args.inputs, args.out)["summary"], + ensure_ascii=False)) + return 0 + if args.cmd == "audio-categories": + from phenomena.audio_categories import write_category_table + document = write_category_table(args.configuration, args.out) + print(json.dumps({"source": document["source"], + "categories": [(row["name"], row["index"], row["volume"]) + for row in document["categories"]]}, + ensure_ascii=False)) + return 0 + if args.cmd == "partvoice-routes": + source, cache = _master_source(args) + if not source: + ap.error("partvoice-routes needs master tables: " + "pass --master or --master-url") + from core.master import Master + from phenomena.partvoice import (build_partvoice_routes, + route_counts, write_partvoice_routes) + roster = json.loads(Path(args.roster).read_text(encoding="utf-8")) + document = build_partvoice_routes(Master(source, cache_dir=cache), + roster) + path = write_partvoice_routes(document, args.out_dir) + print(json.dumps({**route_counts(document), "file": str(path)}, + ensure_ascii=False)) + return 0 + if args.cmd == "material-thumbnails": + from ui.material_thumbnail import export_material_thumbnails + from core.extract import DEFAULT_UNITY_VERSION + result = export_material_thumbnails( + args.bundle, Path(args.master) / "materials.json", args.out_dir, + unity_version=args.unity_version or DEFAULT_UNITY_VERSION) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 if not result["coverage"]["missingMaterialIds"] else 1 + if args.cmd == "honor-images": + from ui.honor_images import export_honor_images + from core.extract import DEFAULT_UNITY_VERSION + result = export_honor_images(args.master, args.bundles, args.out_dir, + unity_version=args.unity_version or DEFAULT_UNITY_VERSION) + print(json.dumps({"counts": result["counts"], "bundles": len(result["bundles"]), + "missingBundles": result["missingBundles"]}, ensure_ascii=False)) + return 0 if not result["missingBundles"] else 1 + if args.cmd == "item-icons": + from ui.item_icon import export_item_icons, package_paths + packages = package_paths(args.decrypted) + masters = {} + if args.root: + for key, name in (("materials", "mysekai-materials.json"), + ("items", "mysekai-items.json"), + ("fixtures", "mysekai-fixtures.json"), + ("tools", "mysekai-tools.json")): + path = Path(args.root) / name + masters[key] = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None + result = export_item_icons(packages, args.out_dir, **masters) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 + if args.cmd == "sitemap-screen-layer": + from sites.screen_layer import extract_screen_layer_jp + result = extract_screen_layer_jp( + args.apk, Path(args.out_dir) / "site" / "sitemap" / "screen_layer", + client_version=args.client_version) + print(json.dumps({"counts": result["counts"], "path": result["path"]}, + ensure_ascii=False)) + return 0 + if args.cmd == "site": + from core.assets.packages import require_paths + from sites.pack import extract_sites + source, cache = _master_source(args) + # Same refusal as the phenomena command: a bare package name under + # --bundle would extract zero objects and report green. + require_paths(args.bundle) + report = extract_sites(args.bundle, args.out_dir, + bundle_root=args.bundle_root, master=source, + master_cache=cache) + print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, + ensure_ascii=False)) + return 0 + if args.cmd == "alone-actions": + from chara.alone_actions import write_alone_actions + print(json.dumps(write_alone_actions(args.bundle, args.out), ensure_ascii=False)) + return 0 + if args.cmd == "facial-tables": + tables = characters.facial_tables(args.bundle) + with open(args.out, "w", encoding="utf-8", newline="\n") as fh: + json.dump(tables, fh, ensure_ascii=False, indent=1, allow_nan=False); fh.write("\n") + print(json.dumps({k: len(v) for k, v in tables.items()})); return 0 + sampled = None + if args.motion_bundle and args.clips: + sampled = characters.sample_clips(args.motion_bundle, [s for s in args.clips.split(",") if s]) + facial = None + if args.facial_tables: + with open(args.facial_tables, encoding="utf-8") as fh: facial = json.load(fh) + cw, ch = args.atlas_cell.lower().split("x") + report = characters.extract_character(args.bundle, args.out_dir, args.name, sampled=sampled, unit_id=args.unit_id, facial=facial, atlas_cell=(int(cw), int(ch)), with_cloth=not args.no_cloth, aux=tuple(args.aux_bundle)) + print(json.dumps(report, ensure_ascii=False)); return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cli/main.py b/src/cli/main.py new file mode 100644 index 0000000..6698481 --- /dev/null +++ b/src/cli/main.py @@ -0,0 +1,18 @@ +"""Grouped public entry point, retaining the existing flat commands.""" +import sys + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] in {"source", "depot"}: + from .pipeline import main as pipeline_main + return pipeline_main(argv) + if argv and argv[0] == "export": + argv = argv[1:] + if argv and argv[0] in {"talks", "communication-actions", "text-catalog", "official-text", "text-runtime"}: + from .pipeline import main as pipeline_main + return pipeline_main(["export", *argv]) + # Existing direct readers stay available through the new command + # group while individual exports move to declared pipeline stages. + from .legacy import main as legacy_main + return legacy_main(argv) diff --git a/src/cli/pipeline.py b/src/cli/pipeline.py new file mode 100644 index 0000000..4699848 --- /dev/null +++ b/src/cli/pipeline.py @@ -0,0 +1,131 @@ +"""Source receipts, reproducible exports and depot inspection.""" +import argparse +import json +from pathlib import Path + +from core.atomic import write_json +from pack.verify import strict_json +from pipeline.depot import Depot +from pipeline.stages import catalog_task, communication_task, official_task, runtime_task, talk_task +from source.snapshot import file_input, read_inputs, snapshot + + +def main(argv): + parser = argparse.ArgumentParser(prog="moly") + groups = parser.add_subparsers(dest="group", required=True) + source = groups.add_parser("source", help="record or verify a versioned source") + sources = source.add_subparsers(dest="command", required=True) + record = sources.add_parser("record") + record.add_argument("--input", action="append", required=True, metavar="ROLE=PATH") + record.add_argument("--region", required=True) + record.add_argument("--version", required=True) + record.add_argument("--out", required=True) + check = sources.add_parser("verify") + check.add_argument("receipt") + export = groups.add_parser("export", help="export products through declared tasks") + exports = export.add_subparsers(dest="command", required=True) + talks = exports.add_parser("talks") + talks.add_argument("--master", required=True) + talks.add_argument("--bundle", required=True) + talks.add_argument("--lib", required=True) + talks.add_argument("--family", choices=("general", "fixture"), default="general") + talks.add_argument("--unity-version", default="2022.3.62f3") + communication = exports.add_parser("communication-actions") + communication.add_argument("--master", required=True) + communication.add_argument("--bundle", required=True) + communication.add_argument("--unity-version", default="2022.3.62f3") + catalog = exports.add_parser("text-catalog") + catalog.add_argument("--corpus", action="append", required=True) + catalog.add_argument("--region", required=True) + catalog.add_argument("--source-revision", required=True) + catalog.add_argument("--previous") + official = exports.add_parser("official-text") + official.add_argument("--source", required=True) + official.add_argument("--donor", required=True) + official.add_argument("--locale", required=True) + runtime = exports.add_parser("text-runtime") + runtime.add_argument("--source", required=True) + runtime.add_argument("--overlay", action="append", default=[]) + for command in (talks, communication, catalog, official, runtime): + command.add_argument("--depot", required=True) + command.add_argument("--out", required=True, help="layout directory, outside the depot") + command.add_argument("--plan", action="store_true", help="inspect reuse without extraction or layout") + command.add_argument("--emit-task", help="save the portable task description") + command.add_argument("--force", action="store_true", help="recompute and compare the existing result") + depot = groups.add_parser("depot", help="inspect, run, verify and lay out tasks") + depots = depot.add_subparsers(dest="command", required=True) + for name in ("plan", "run", "verify", "layout", "gc"): + command = depots.add_parser(name) + command.add_argument("--depot", required=True) + if name in ("plan", "run"): + command.add_argument("--task", required=True) + if name == "plan": + command.add_argument("--previous", help="previous task for invalidation reasons") + if name == "run": + command.add_argument("--force", action="store_true") + if name == "layout": + command.add_argument("--key", required=True) + command.add_argument("--out", required=True) + if name == "gc": + command.add_argument("--apply", action="store_true") + args = parser.parse_args(argv) + try: + if args.group == "source": + if args.command == "record": + inputs = [] + for item in args.input: + role, separator, path = item.partition("=") + if not separator or not role or not path: + raise ValueError("--input expects ROLE=PATH") + inputs.append(file_input(role, path)) + result = snapshot(inputs, region=args.region, version=args.version) + write_json(args.out, result) + report = {"revision": result["revision"], "inputs": len(inputs)} + else: + result = strict_json(Path(args.receipt).read_bytes()) + rebuilt = snapshot(result["inputs"], region=result["region"], version=result["version"]) + if result != rebuilt: + raise ValueError("source receipt identity differs from its inputs") + read_inputs(result["inputs"]) + report = {"revision": result["revision"], "verified": True} + elif args.group == "export": + if args.command == "talks": + task = talk_task(args.master, args.bundle, args.lib, family=args.family, + unity_version=args.unity_version) + elif args.command == "communication-actions": + task = communication_task(args.master, args.bundle, unity_version=args.unity_version) + elif args.command == "text-catalog": + task = catalog_task(args.corpus, region=args.region, source_revision=args.source_revision, + previous=args.previous) + elif args.command == "official-text": + task = official_task(args.source, args.donor, locale=args.locale) + else: + task = runtime_task(args.source, args.overlay) + depot = Depot(args.depot) + if args.emit_task: + write_json(args.emit_task, task) + if args.plan: + report = depot.plan(task) + else: + run = depot.run(task, force=args.force) + report = {"status": run["status"], **depot.materialize(task["key"], args.out)} + else: + depot = Depot(args.depot) + if args.command in {"plan", "run"}: + task = strict_json(Path(args.task).read_bytes()) + if args.command == "plan": + previous = strict_json(Path(args.previous).read_bytes()) if args.previous else None + report = depot.plan(task, previous=previous) + else: + report = depot.run(task, force=args.force) + elif args.command == "layout": + report = depot.materialize(args.key, args.out) + elif args.command == "gc": + from pack.gc import collect_garbage + report = collect_garbage(args.depot, apply=args.apply) + else: + report = depot.verify() + print(json.dumps(report, ensure_ascii=False)) + return 0 + except (OSError, ValueError, KeyError, TypeError) as exc: + parser.exit(2, f"moly: {exc}\n") diff --git a/src/core/assets/router.py b/src/core/assets/router.py index cc22c25..d84ade9 100644 --- a/src/core/assets/router.py +++ b/src/core/assets/router.py @@ -187,6 +187,8 @@ def route(name): return Route("facial", "facial-tables") if name == "mysekai__character_alone_action": return Route("performance", "alone-actions") + if name == "mysekai__some_character_communication": + return Route("communication", "communication-actions") if "__effect__emoticon__" in name: return Route("emoticon", "emoticons") if name == "mysekai__talk__scenario__talk": diff --git a/src/core/cli.py b/src/core/cli.py index 7e5378c..1eea1aa 100644 --- a/src/core/cli.py +++ b/src/core/cli.py @@ -1,1688 +1,7 @@ -"""Single command-line entry point for bundle extraction.""" -import argparse -import os -import warnings -import json +"""Compatibility entry point; command implementation lives in cli.""" import sys -from pathlib import Path - -# Extraction output root; MOLY_EXTRACT_OUT lets a workspace keep it outside the checkout. -DEFAULT_DATA_ROOT = os.environ.get("MOLY_EXTRACT_OUT") or "local-data" - - -PLAYER_DATA_HELP = ("the player data: the container file (assets/bin/Data/data.unity3d), " - "or, for a build without one, each APK whose assets/bin/Data members " - "are its loose player files; repeat for the base APK and the asset pack") - - -def _configure_unity(fallback_version): - import UnityPy.config - if fallback_version: - UnityPy.config.FALLBACK_UNITY_VERSION = fallback_version - - -def _master_source(args): - """Directory or base URL for the caller's master tables, plus a cache dir. - - ``--master`` names a local directory; ``--master-url`` fetches each table by - appending its name to a base URL (with no value, the public default base). - Neither is implied: without one of them nothing is read and nothing is - fetched. - """ - from .master import DEFAULT_MASTER_URL - url = getattr(args, "master_url", None) - if url is not None: - return url or DEFAULT_MASTER_URL, getattr(args, "master_cache", None) - return getattr(args, "master", None), None - - -def _write_programs(directory, entries, platform=None): - """Write each program body to its own file; return how many were written. - - A body is named by shader, platform and record index rather than by content - hash, so a caller reading a file knows which variant produced it; identical - variants therefore appear more than once, which is the honest shape -- the - census reports the distinct count separately. - """ - directory = Path(directory) - directory.mkdir(parents=True, exist_ok=True) - written = 0 - for entry in entries: - stem = "".join(c if c.isalnum() or c in "-_" else "_" for c in entry["shader"]) - for block in entry["platforms"]: - if platform is not None and block["platform"] != platform: - continue - for row in block["programs"]: - suffix = "glsl" if row["isText"] else "bin" - name = f"{stem}__platform{block['platform']}__record{row['record']}.{suffix}" - (directory / name).write_bytes(row.get("code") or b"") - written += 1 - return written - - -def _extract_failed(report): - """A failed derived artifact is a failed extraction, not a skipped option. - - So is a failed player-data artifact: its input was supplied and it could - not be read, which is a failure of the run, while a player-data artifact - whose input nobody supplied stays ``skipped`` and is named as an input gap. - """ - return bool(report["summary"]["failed"]) or any( - entry.get("status") == "failed" - for entry in report.get("derived", []) + report.get("playerData", [])) - - -def _print_extract_summary(report, out_dir): - s = report["summary"] - print(f"extracted {s['succeeded']}/{s['requested']} bundles -> {out_dir} " - f"(failed {s['failed']}, unsupported {s['unsupported']})") - for entry in report["bundles"]: - if entry["status"] == "failed": - print(f" failed: {entry['bundle']}: {entry['error']}") - for entry in report.get("derived", []) + report.get("playerData", []): - if entry.get("status") == "failed": - print(f" failed artifact: {entry['artifact']}: {entry.get('error', '')}") - programs = entry.get("sourcePrograms") - if programs: - print(f" particle source programs: {programs['status']} " - f"({programs['converted']}/{programs['gles3Variants']} GLES3 variants converted, " - f"unavailable {programs['unavailable']})") - print(f" {programs['note']}") - # An artifact whose input nobody supplied is not a failure of the run, but - # it is not silent either: each one is named with its reason (a failed one - # is already named above). - for gap in (report.get("inputGaps") or {}).get("unaccepted", []): - if gap.get("why") != "failed": - print(f" input gap: {gap['artifact']}: {gap.get('detail', '')}") - print(f"report: {report['report']}") - if str(out_dir) == "local-data": - print("view: python -m http.server 8000 (from the repository root), then open\n" - " http://localhost:8000/examples/viewer/index.html?base=../../local-data") - else: - print("view: serve the repository root over HTTP and open " - "examples/viewer/index.html?base=") - - -def main(argv=None): - from chara.rotation_encoding import DEFAULT_ROTATION_ENCODING, ROTATION_ENCODINGS - # UnityPy warns about asset shapes it does not model. Silencing that is - # this command's choice to make, so it is made here, once, for this - # process. It used to sit at module scope in four extractors, where - # importing one rewrote the warning filters of any program that used this - # package as a library -- including filters that program had set itself. - warnings.filterwarnings("ignore") - ap = argparse.ArgumentParser(prog="moly", description="Extract Unity humanoid assets") - ap.add_argument("--unity-version", default=None) - sub = ap.add_subparsers(dest="cmd", required=True) - p = sub.add_parser("pull", help="download, decrypt, and extract character asset packs") - p.add_argument("--manifest", required=True) - p.add_argument("--out", help="download/decrypt workspace (default: moly-pull-output)") - p.add_argument("--extract-out", default=None, - help="where extraction artifacts go (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") - p.add_argument("--json", action="store_true", - help="print the full JSON report to stdout instead of a summary") - p.add_argument("--asset-base-url", required=True) - p.add_argument("--workers", type=int, default=8) - p.add_argument("--retries", type=int, default=4) - p.add_argument("--master", help="directory of caller-supplied master tables") - p.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - p.add_argument("--master-cache", help="where fetched tables are cached") - p.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - p.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") - x = sub.add_parser("extract", help="extract bundles listed in a manifest") - x.add_argument("--manifest", - help="optional bundle list; omitted = select every supported " - "asset-pack bundle found under --bundles") - x.add_argument("--bundles", required=True) - x.add_argument("--out", default=DEFAULT_DATA_ROOT, - help="output directory (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") - x.add_argument("--json", action="store_true", - help="print the full JSON report to stdout instead of a summary") - x.add_argument("--master", help="directory of caller-supplied master tables") - x.add_argument("--player-data", action="append", - help="the APK player data: the container file " - "(assets/bin/Data/data.unity3d), or, for a build " - "without one, each APK whose assets/bin/Data members " - "are its loose player files (repeat: the base APK and " - "the asset pack). It supplies the built-in UI and " - "screen-layer assets that the download path does not " - "carry (container builds only) and the Resources " - "objects: the live transitioner, the effect type " - "table, the graphics configuration, the layout grid") - x.add_argument("--player-types", metavar="DIR", - help="managed assembly stubs of the same player build (a DummyDll " - "directory); the Resources objects read from --player-data " - "(the live transitioner, the effect type table, the drop-item " - "shading) need them, and without them those artifacts are " - "skipped with the reason") - x.add_argument("--player-declarations", metavar="FILE", - help="the same build's declaration dump; names the effect types in " - "effect-resources/ (without it the names are null)") - x.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - x.add_argument("--master-cache", help="where fetched tables are cached") - x.add_argument("--fixture-meshes", action="store_true", - help="also write furniture geometry (fixture-models/), one " - "glTF binary per package; without it that pass is " - "skipped and the passes that need the geometry report " - "it as a missing dependency") - x.add_argument("--fixture-particles", action="store_true", - help="also write furniture particle emitters " - "(fixture-particles/), one document per package " - "across the fixture-interface, cutscene-timeline and " - "fixture-timeline families; without it that pass is " - "skipped. Playable output requires --shader-compiler; " - "without it only source data is saved, not playable assets") - x.add_argument("--shader-compiler", metavar="PATH", - help="pinned offline source shader compiler required for playable " - "furniture particles; omitting it saves source-only data") - x.add_argument("--builtin-resources", action="append", default=[], - metavar="PATH", - help="the engine's own built-in resource container, or a " - "directory holding it; some particle-system renderers " - "and some phenomena emitters draw a copy of a " - "built-in primitive mesh and no package ships those. " - "Repeatable. Without it those mesh slots stay listed " - "as unresolved in fixture-particles/ and in the " - "phenomena index, exactly as before") - x.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - x.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") - x.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") - c = sub.add_parser("characters", help="extract one character bundle") - c.add_argument("--bundle", required=True); c.add_argument("--out-dir", required=True); c.add_argument("--name", required=True) - c.add_argument("--aux-bundle", action="append", default=[]); c.add_argument("--motion-bundle"); c.add_argument("--clips"); c.add_argument("--unit-id", type=int); c.add_argument("--facial-tables"); c.add_argument("--atlas-cell", default="512x256"); c.add_argument("--no-cloth", action="store_true") - m = sub.add_parser("motion-library", help="export the shared humanoid motion library") - m.add_argument("--reference-bundle", required=True); m.add_argument("--motion-bundle", required=True); m.add_argument("--aux-bundle", action="append", default=[]); m.add_argument("--clips"); m.add_argument("--out-dir", required=True); m.add_argument("--name", default="motion-library") - m.add_argument("--controller-bundle", action="append", default=None, - help="character package whose AnimatorController selects the clips " - "(by serialized identity); repeat per package. Without it the " - "first clip of each name is kept and duplicates are listed") - m.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="rotation key storage (default: snorm16); float32 disables lossy encoding") - t = sub.add_parser("facial-tables", help="dump facial tables") - t.add_argument("--bundle", required=True); t.add_argument("--out", required=True) - n = sub.add_parser("alone-actions", help="dump per-character performance data (motion x facial x timing)") - n.add_argument("--bundle", required=True); n.add_argument("--out", required=True) - g = sub.add_parser("registry", help="build the character registry from master tables") - g.add_argument("--master", help="directory of caller-supplied master tables") - g.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - g.add_argument("--master-cache", help="where fetched tables are cached") - g.add_argument("--manifest", required=True, help="manifest naming the character bundles") - g.add_argument("--motion-index", help="shared motion library index, to check motion names") - g.add_argument("--out", required=True) - - k = sub.add_parser("talks", help="extract single-character direct-talk corpus") - k.add_argument("--master", help="directory of caller-supplied master tables") - k.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - k.add_argument("--master-cache", help="where fetched tables are cached") - k.add_argument("--bundle", required=True, help="talk scenario bundle") - k.add_argument("--lib", help="decrypted lib package carrying the constant " - "tables the scripts name; without it every " - "Table.key token stays a source token") - k.add_argument("--out", required=True) - - tw = sub.add_parser("tweets", help="extract tweet texts and per-character after-edit pools") - tw.add_argument("--master", help="directory of caller-supplied master tables") - tw.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - tw.add_argument("--master-cache", help="where fetched tables are cached") - tw.add_argument("--out", required=True) - - tt = sub.add_parser("tweet-tables", - help="extract the tweet selection tables (greetings, " - "greeting conditions, site entries, talk pre-actions)") - tt.add_argument("--master", help="directory of caller-supplied master tables") - tt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - tt.add_argument("--master-cache", help="where fetched tables are cached") - tt.add_argument("--out", required=True) - - sg = sub.add_parser("site-groups", - help="extract the mysekai site-group table " - "(siteGroupId -> the site ids it holds)") - sg.add_argument("--master", help="directory of caller-supplied master tables") - sg.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - sg.add_argument("--master-cache", help="where fetched tables are cached") - sg.add_argument("--out", required=True) - - bp = sub.add_parser("birthday-parties", - help="extract the birthday-party campaign table the " - "site-map festival gate reads (startAt/closedAt windows)") - bp.add_argument("--master", help="directory of caller-supplied master tables") - bp.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bp.add_argument("--master-cache", help="where fetched tables are cached") - bp.add_argument("--out", required=True) - - bd = sub.add_parser("birthday-party-delivery", - help="extract the birthday-party delivery tables the " - "delivery site reads (rewards, point bonuses, total rewards)") - bd.add_argument("--master", help="directory of caller-supplied master tables") - bd.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bd.add_argument("--master-cache", help="where fetched tables are cached") - bd.add_argument("--out", required=True) - - gv = sub.add_parser("gate-visit-policy", - help="extract the gate visit tables as policy data for the " - "server mock (not a client character input)") - gv.add_argument("--master", help="directory of caller-supplied master tables") - gv.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - gv.add_argument("--master-cache", help="where fetched tables are cached") - gv.add_argument("--out", required=True) - - tm = sub.add_parser("time-manager", - help="extract the project TimeManager settings (fixed step, " - "maximum allowed timestep, time scale, particle step) " - "from an APK or a globalgamemanagers file") - tm.add_argument("--source", required=True, - help="an APK, or a player-data globalgamemanagers file") - tm.add_argument("--out", required=True) - - cc = sub.add_parser("client-config", - help="extract the ClientConfig deliverable panel " - "(the four typed dictionaries, keyed by id)") - cc.add_argument("--master", help="directory of caller-supplied master tables") - cc.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - cc.add_argument("--master-cache", help="where fetched tables are cached") - cc.add_argument("--out", required=True) - - from core.master_tables import REGISTERED_TABLES - mt = sub.add_parser("master-tables", - help="extract selected key-addressed mysekai master " - "tables, defaulting to blueprints, items, music " - "records and wordings") - mt.add_argument("--master", help="directory of caller-supplied master tables") - mt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - mt.add_argument("--master-cache", help="where fetched tables are cached") - mt.add_argument("--table", dest="tables", action="append", - choices=tuple(REGISTERED_TABLES), - help="registered table to export; repeat to select more " - "(omitting this exports only the original four tables)") - mt.add_argument("--out", required=True, - help="directory the selected documents are written into") - - e = sub.add_parser("emoticons", help="extract overhead-item effect packages") - e.add_argument("--bundle", action="append", required=True); e.add_argument("--out-dir", required=True) - v = sub.add_parser("avatar-parts", help="extract player-appearance packages (skin/decoration/penlight)") - v.add_argument("--bundle", action="append", required=True); v.add_argument("--out-dir", required=True) - ht = sub.add_parser("harvest-tools", help="extract original harvest-tool prefabs and their material data") - ht.add_argument("--bundle", action="append", required=True) - ht.add_argument("--bundle-root", help="directory containing declared shader dependencies") - ht.add_argument("--out-dir", required=True, help="tool directory, normally /avatar/tools") - ptm = sub.add_parser("player-tool-models", - help="extract the avatar's hand-item prefabs and their material data " - "(mysekai/player_tool_model/*)") - ptm.add_argument("--bundle", action="append", required=True) - ptm.add_argument("--bundle-root", help="directory containing declared shader dependencies") - ptm.add_argument("--out-dir", required=True, - help="model directory, normally /avatar/player-tool-models") - er = sub.add_parser("effect-resources", - help="export the effect manager's type table " - "(Resources Mysekai/Effects/EffectResourceData) from player data") - er.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - er.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - er.add_argument("--declarations", - help="the same build's declaration dump, for effect type names") - er.add_argument("--bundle-root", - help="bundle directory to check each row's bundle and prefab against") - er.add_argument("--out", required=True, help="output JSON path") - dis = sub.add_parser("drop-item-shade", - help="export the drop-item shading settings of the graphics " - "configuration (Resources Graphics/Data/MysekaiGraphicsConfig)") - dis.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - dis.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - dis.add_argument("--out", required=True, help="output JSON path") - sts = sub.add_parser("stencil-shadow", - help="export the character stencil-shadow colour and the material " - "rendering settings of the graphics configuration") - sts.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - sts.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - sts.add_argument("--out", required=True, help="output JSON path") - lgr = sub.add_parser("layout-grid", - help="export the layout editor's placement grid: its settings, " - "materials, shader programs and colour profiles " - "(Resources Data/MysekaiGridSettings and " - "Data/GridColorProfileTable)") - lgr.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - lgr.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - lgr.add_argument("--out-dir", required=True, - help="output directory, normally /layout-grid") - rpf = sub.add_parser("resource-prefab", - help="extract one prefab the player data ships in Resources, by load name") - rpf.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - rpf.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - rpf.add_argument("--load-name", required=True, help="the Resources.Load path") - rpf.add_argument("--out-dir", required=True, - help="directory receiving /.{json,glb} and textures") - rpf.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", - help="the engine's built-in resource file (unity default resources, " - "unity_builtin_extra) or a directory holding them; a container " - "build keeps them beside data.unity3d; repeat for each") - rpr = sub.add_parser("resource-programs", - help="write the compiled programs of every shader a Resources " - "prefab's materials use (the program library layout)") - rpr.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - rpr.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - rpr.add_argument("--load-name", required=True, help="the Resources.Load path") - rpr.add_argument("--out-dir", required=True, - help="directory receiving /shaders/ (the resource-prefab out-dir)") - rpr.add_argument("--shader-compiler", - help="offline shader compiler executable; without it the programs are " - "written as source only (no conversions)") - fp = sub.add_parser("fixture-particles", - help="write particle source-schema documents (the fixture-particles " - "layout) for explicitly named bundles") - fp.add_argument("--bundle", action="append", required=True) - fp.add_argument("--bundle-root", help="directory the bundles' dependencies are read from") - fp.add_argument("--domain", action="append", - help="routed domain to extract; repeat; default the three furniture domains") - fp.add_argument("--shader-compiler", metavar="PATH", - help="pinned offline shader compiler; without it programs stay source-only") - fp.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", - help="the engine's built-in resource container (or its directory), " - "for renderers that point at a built-in mesh") - fp.add_argument("--out-dir", required=True) - pp = sub.add_parser("prefab-package", - help="extract one named prefab of a bundle with the site package reader") - pp.add_argument("--bundle", required=True, help="the bundle file") - pp.add_argument("--prefab", required=True, - help="the prefab's container basename, e.g. fieldcamera.prefab") - pp.add_argument("--bundle-root", help="directory the bundle's dependencies are read from") - pp.add_argument("--out-dir", required=True, - help="directory receiving /.{json,glb} and textures") - sae = sub.add_parser("site-action-effects", - help="extract the site-move and harvest-action effect prefabs " - "(mysekai/effect/site/harvest/action/*)") - sae.add_argument("--bundle", action="append", required=True) - sae.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - sae.add_argument("--out-dir", required=True, - help="effect directory, normally /site-action-effects") - she = sub.add_parser("site-home-effects", - help="extract the home-site effect prefabs, the craft effect among " - "them (mysekai/effect/site/home/*)") - she.add_argument("--bundle", action="append", required=True) - she.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - she.add_argument("--out-dir", required=True, - help="effect directory, normally /site-home-effects") - fpe = sub.add_parser("fixture-put-effects", - help="extract the furniture placement effect prefab " - "(mysekai/effect/fixture/fx_fixture_put_01)") - fpe.add_argument("--bundle", action="append", required=True) - fpe.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - fpe.add_argument("--out-dir", required=True, - help="effect directory, normally /fixture-put-effects") - ssw = sub.add_parser("site-sound-wave", - help="resolve MysekaiSiteView._soundWaveMeshFilter (the footstep " - "surface) of site scene packages: node, mesh and vertex colours") - ssw.add_argument("--bundle", action="append", required=True, - help="a mysekai__site__field__ package; repeat") - ssw.add_argument("--bundle-root", help="directory the scene packages' dependencies are read from") - ssw.add_argument("--out", required=True, help="output JSON path") - sce = sub.add_parser("site-common-action-effects", - help="extract the player's foot effect prefabs " - "(mysekai/effect/site/common/action/*)") - sce.add_argument("--bundle", action="append", required=True) - sce.add_argument("--bundle-root", - help="directory containing declared effect, texture and shader dependencies") - sce.add_argument("--out-dir", required=True, - help="effect directory, normally /site-common-action-effects") - sho = sub.add_parser("site-harvest-object-effects", - help="extract the harvest-object hit, delete and stay effect prefabs " - "(mysekai/effect/site/harvest/object//*)") - sho.add_argument("--bundle", action="append", required=True) - sho.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - sho.add_argument("--out-dir", required=True, - help="effect directory, normally /site-harvest-object-effects") - pa = sub.add_parser("player-avatar", - help="export the player avatar composite (audience model + skeleton + motion clips) as one glb") - pa.add_argument("--bundle", action="append", required=True, - help="the avatar model bundle first, then every motion bundle whose clips " - "are to be resolved against it (the order export_player_avatar takes); " - "repeat --bundle per package") - pa.add_argument("--out-dir", required=True) - pa.add_argument("--name", default="mysekai__player_avatar", - help="output basename (glb + .index.json + .motion-manifest.json)") - w = sub.add_parser("phenomena", help="extract weather (phenomena) environment packages") - w.add_argument("--bundle", action="append", required=True, - help="an environment package, or the shared phenomena thumbnail package") - w.add_argument("--bundle-root", - help="directory the packages a bundle depends on are read from") - w.add_argument("--builtin-resources", action="append", default=[], - metavar="PATH", - help="the engine's own built-in resource container, or a " - "directory holding it; some emitters draw copies of a " - "built-in primitive and no package ships those. Repeatable. " - "Without it those meshes stay listed under `unsupported`, " - "exactly as before") - w.add_argument("--shader-compiler", metavar="PATH", - help="moly-shader-bridge executable; without it source programs " - "are preserved and their conversion is explicitly unavailable") - w.add_argument("--site-bundle", action="append", default=None, metavar="PATH", - help="a site scene package whose site prefab colliders go into " - "collision.json; repeatable. Without it every scene package " - "under --bundle-root is used") - w.add_argument("--out-dir", required=True) - w.add_argument("--master", help="directory of caller-supplied master tables") - w.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - w.add_argument("--master-cache", help="where fetched tables are cached") - w.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it; " - "without it PATH is searched and the audio " - "entry says what is missing") - w.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - a = sub.add_parser("mysekai-audio", - help="extract the rest of the sound corpus: talk voices " - "named by talks.json, plus the remaining se, bgm " - "and part-voice packages") - a.add_argument("--talks", required=True, - help="a talk corpus in either shape this repo's extractors " - "write (talks grouped under units, or a flat talks " - "list); its voice cues are the talk-voice denominator, " - "and only those cues are decoded") - a.add_argument("--manifest", required=True, - help="asset bundle manifest giving the family denominators") - a.add_argument("--bundle-root", required=True, - help="directory holding the decrypted sound packages") - a.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "the products and whose loop.json is merged into") - a.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - a.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - ca = sub.add_parser("cutscene-audio", - help="decode the cues a cut-scene's own timeline names: its " - "voice clips from the cut-scene's birthday voice bank, " - "its music and effect clips from the packages they name") - ca.add_argument("--clips", required=True, action="append", - help="a cut-scene clip document (cutscene-timeline/clips/" - ".json); repeat for each cut-scene") - ca.add_argument("--cut-scenes", required=True, - help="the cut-scene master table (mysekaiCutScenes.json), " - "which routes voice clips to their voice bank") - ca.add_argument("--manifest", required=True, - help="asset bundle manifest the named packages are checked against") - ca.add_argument("--bundle-root", required=True, - help="directory holding the decrypted sound packages") - ca.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "the products and whose loop.json and corpus.json are merged into") - ca.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - ca.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - sq = sub.add_parser("audio-sequences", - help="attach the cues' sequence structure (type, repeat, " - "per-track delay, track -> stream) to an existing " - "loop sidecar, reading the archives again from the " - "decrypted tree; writes a copy, never in place") - sq.add_argument("--loop", required=True, help="the loop sidecar to read") - sq.add_argument("--bundle-root", required=True, action="append", - help="directory holding the decrypted sound packages; repeat " - "to give further trees, tried in order per entry") - sq.add_argument("--out", required=True, - help="where the annotated copy is written (must not be --loop)") - ra = sub.add_parser("record-audio", - help="read the waveform facts (length, loop region or none, " - "cue commands) of every package a music record can " - "play; the audio itself is not written") - ra.add_argument("--master", help="directory of caller-supplied master tables") - ra.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - ra.add_argument("--master-cache", help="where fetched tables are cached") - ra.add_argument("--bundle-root", required=True, action="append", - help="directory holding decrypted bundles; repeat to give further trees") - ra.add_argument("--bundle-version", required=True, - help="the asset version the bundles were downloaded at (written on every row)") - ra.add_argument("--only", choices=("music", "music_sound_track"), - help="read only the packages of this record track type") - ra.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - ra.add_argument("--out", required=True, help="the document to write") - rs = sub.add_parser("record-audio-storage", - help="add each row's storage mirror audio file: sha256 and size of " - "the downloaded bytes and the listing's version") - rs.add_argument("input", help="a record-audio document") - rs.add_argument("--server", required=True, choices=("jp", "cn")) - rs.add_argument("--workers", type=int, default=4) - rs.add_argument("--out", required=True) - rm = sub.add_parser("record-audio-merge", - help="combine record-audio documents of separate runs") - rm.add_argument("inputs", nargs="+") - rm.add_argument("--out", required=True) - rc = sub.add_parser("record-audio-carry", - help="carry a read record-audio document's facts to another " - "region's packages whose bundles have the same crc and size") - rc.add_argument("--master", help="directory of the target region's master tables") - rc.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - rc.add_argument("--master-cache", help="where fetched tables are cached") - rc.add_argument("--manifest", required=True, - help="the target region's package list (AssetBundleInfoNew)") - rc.add_argument("--source", required=True, - help="a record-audio document with storage attached") - rc.add_argument("--source-manifest", required=True, - help="the package list of the bundles the source document was read from") - rc.add_argument("--region", required=True, choices=("jp", "cn"), - help="the target region, written on the document") - rc.add_argument("--out", required=True) - rf = sub.add_parser("record-audio-fetch", - help="download and check (size, uncompressed crc) the bundles of " - "the packages a record-audio document lists as absent") - rf.add_argument("document", help="a record-audio document (its absent list is fetched)") - rf.add_argument("--manifest", required=True, - help="the package list (AssetBundleInfoNew) the bundles are fetched by") - rf.add_argument("--asset-base-url", required=True, - help="the prefix before each entry's downloadPath") - rf.add_argument("--workers", type=int, default=4) - rf.add_argument("--out", required=True, help="workspace; bundles land in /decrypted") - ac = sub.add_parser("audio-categories", - help="write the sound configuration's category table (name, " - "index, volume) the runtime scales cues by") - ac.add_argument("--configuration", required=True, - help="the sound configuration (.acf), or the application " - "package (zip) that holds assets/*.acf") - ac.add_argument("--out", required=True, - help="where categories.json is written (the runtime reads " - "phenomena/audio/categories.json)") - r = sub.add_parser("partvoice-routes", - help="build the speaker-to-partvoice-package routing " - "table from master tables and the roster") - r.add_argument("--master", help="directory of caller-supplied master tables") - r.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - r.add_argument("--master-cache", help="where fetched tables are cached") - r.add_argument("--roster", required=True, - help="characters.json, the registry artifact; its " - "characters are the participants' universe and order") - r.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "partvoice.json") - u = sub.add_parser("ui-action-icons", - help="export the standalone action-button icon bundles " - "as one PNG per bundle plus a manifest") - u.add_argument("--bundle", action="append", required=True, - help="an action-icon bundle; repeat for as many as wanted") - u.add_argument("--out-dir", required=True) - portraits = sub.add_parser("browser-assets", - help="export source character portraits for the browser catalog") - portraits.add_argument("--player-data", required=True, - help="the APK player data containing CharacterSDIconAtlas") - portraits.add_argument("--asset-root", required=True, - help="runtime resource root containing characters.json") - portraits.add_argument("--source-region", required=True, choices=("cn", "jp")) - portraits.add_argument("--source-game-version", required=True, - help="game version of the supplied UI player data") - presentation = sub.add_parser("fixture-presentation", - help="resolve model, wallpaper, floor and custom furniture source routes") - presentation.add_argument("--asset-root", required=True) - presentation.add_argument("--master", required=True) - presentation.add_argument("--bundles", required=True) - validate = sub.add_parser("validate-runtime", help="verify referenced resources and exact audio cue coverage") - validate.add_argument("--asset-root", required=True) - validate.add_argument("--verify-actor-hashes", action="store_true") - fixture_animations = sub.add_parser("fixture-animations", - help="refresh source-bound fixture clips and their authored time domain") - fixture_animations.add_argument("--asset-root", required=True) - fixture_animations.add_argument("--bundles", required=True) - fixture_animations.add_argument("--package", action="append") - fixture_animations.add_argument("--plan", action="store_true") - capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") - capabilities.add_argument("--asset-root", required=True) - lay = sub.add_parser("ui-layout", - help="extract the RectTransform hierarchy of the UI " - "prefabs (screen layers, dialogs) the player data " - "builds, plus a census of the UI package families") - lay.add_argument("--region", choices=("cn", "jp"), default="cn", - help="cn: one player data file whose resources.assets holds " - "every prefab; jp: the base APK and the asset pack, " - "whose Resources index points at one serialized file " - "per prefab") - lay.add_argument("--player-data", default=None, - help="cn: the APK player data file; every screen and dialog " - "prefab lives in its resources.assets (Resources.Load " - "fetches them, so no bundle name routes to them)") - lay.add_argument("--apk", action="append", default=None, - help="jp: an APK whose assets/bin/Data members are player " - "files; give the base APK and the asset pack") - lay.add_argument("--client-version", default=None, - help="jp: the client version the manifest and every document " - "name (ui/jp_decoders.py names the supported ones; each " - "object's chain comes from its serialized type hash); cn: " - "the client version the palette and tween defaults name") - lay.add_argument("--master", default=None, - help="jp: directory of master tables holding wordings.json") - lay.add_argument("--prefab", action="append", default=None, - help="a prefab name to extract (repeatable); a targeted " - "run writes those documents and omits the broad census") - lay.add_argument("--out-dir", required=True, - help="the output directory (ui-layout/) that receives one " - "document per prefab plus census.json") - lay.add_argument("--bundles", default=None, - help="optional decrypted-package directory; named so the " - "census also reports what the per-screen download " - "packages contain (textures and effects, not layout)") - lay.add_argument("--font-bundles", default=None, - help="jp: decrypted-package directory holding font/common " - "(jp download --root font/common); defaults to --bundles") - lay.add_argument("--bundle-manifest", default=None, - help="optional AssetBundleInfoNew.json; supplies the " - "isBuiltin split and declared sizes for the census") - words = sub.add_parser("ui-wordings", - help="write the runtime wording dictionary of a container build: " - "the built-in wording CSV of its language, then the master " - "wordings over it (the document a shared root reads as its " - "wordings.json)") - words.add_argument("--player-data", required=True, - help="the APK player data file whose Resources index holds the wording CSV") - words.add_argument("--master", required=True, - help="directory of master tables holding wordings.json") - words.add_argument("--client-version", required=True, - help="the client version the document's source block names") - words.add_argument("--out", required=True, help="the document path") - reactions = sub.add_parser("fixture-reactions", - help="export the native furniture reaction-character groups") - reactions.add_argument("--bundle", required=True, - help="decrypted mysekai__system__fixture_reaction_data bundle") - reactions.add_argument("--out", required=True, - help="output fixture-reactions.json path") - bem = sub.add_parser("birthday-end-motions", - help="export the waving motion the birthday party " - "plays as it closes (row per character unit)") - bem.add_argument("--bundle", required=True, - help="decrypted mysekai__birthday__birthday_end_character_motion_data_list bundle") - bem.add_argument("--out", required=True, - help="output birthday-end-character-motions.json path") - bpt = sub.add_parser("birthday-party-talks", - help="attach the birthday-party scripts one master " - "table names to an existing talks.json") - bpt.add_argument("--master", help="directory of caller-supplied master tables") - bpt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bpt.add_argument("--master-cache", help="where fetched tables are cached") - bpt.add_argument("--bundle", required=True, help="talk scenario bundle") - bpt.add_argument("--lib", help="decrypted lib package carrying the constant " - "tables the scripts name") - bpt.add_argument("--talks", required=True, - help="an existing talks.json whose birthdayParty block is " - "computed from this master partition") - bpt.add_argument("--out", required=True) - f = sub.add_parser("fixture-master-slice", - help="slice mysekaiFixtures down to the columns the " - "runtime reads (button gating and footprint)") - f.add_argument("--master", help="directory of caller-supplied master tables") - f.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - f.add_argument("--master-cache", help="where fetched tables are cached") - f.add_argument("--out", required=True) - q = sub.add_parser("site", help="extract the site (place) asset packages") - q.add_argument("--bundle", action="append", required=True, - help="a package under the site path; repeat for as many as wanted") - q.add_argument("--bundle-root", - help="directory the packages a bundle depends on are read from") - q.add_argument("--out-dir", required=True) - q.add_argument("--master", help="directory of caller-supplied master tables") - q.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - q.add_argument("--master-cache", help="where fetched tables are cached") - ua = sub.add_parser("ui-audio", help="extract embedded UI cue sheets with the shared audio decoder") - ua.add_argument("--apk", required=True) - ua.add_argument("--out", required=True) - ua.add_argument("--vgmstream") - ua.add_argument("--ffmpeg") - sls = sub.add_parser("sitemap-screen-layer", - help="write site/sitemap/screen_layer/screen_layer.json from APK " - "members (player data without resources.assets)") - sls.add_argument("--apk", action="append", required=True, - help="an APK whose assets/bin/Data members are player files; " - "give the base APK and the asset pack") - sls.add_argument("--client-version", required=True, - help="the client version the field chains are for") - sls.add_argument("--out-dir", required=True, - help="runtime root; the document lands at " - "site/sitemap/screen_layer/screen_layer.json under it") - mth = sub.add_parser("material-thumbnails", - help="write material-thumbnails.json and textures/material{id}.png " - "from the thumbnail/material package, one per materials row") - mth.add_argument("--bundle", required=True, help="the decrypted thumbnail/material package") - mth.add_argument("--master", required=True, - help="directory of master tables holding materials.json") - mth.add_argument("--out-dir", required=True, - help="output directory (material-thumbnails/ of a runtime root)") - hon = sub.add_parser("honor-images", - help="write honor/index.json and the textures of the honor bundles " - "the birthday-party delivery rewards' honors draw") - hon.add_argument("--master", required=True, - help="directory of master tables (the delivery total rewards, resource boxes, honors, honor groups)") - hon.add_argument("--bundles", required=True, help="directory of decrypted honor and honor_frame packages") - hon.add_argument("--out-dir", required=True, help="runtime root; images land under honor/ and honor_frame/") - ipv = sub.add_parser("item-icons", - help="write index.json and textures///.png from " - "the mysekai item_preview and thumbnail material/item/tool packages, " - "keyed by the package path") - ipv.add_argument("--decrypted", required=True, - help="directory holding the decrypted mysekai item icon packages") - ipv.add_argument("--root", default=None, - help="runtime root whose mysekai-materials/items/fixtures/tools documents " - "give the coverage report") - ipv.add_argument("--out-dir", required=True, - help="output directory (mysekai-item-icons/ of a runtime root)") - a = sub.add_parser("fetch-apk", help="discover, download, or inspect an Android APK") - a.add_argument("--endpoint", default=None); a.add_argument("--timeout", type=float, default=30.0); a.add_argument("--retries", type=int, default=3) - asub = a.add_subparsers(dest="apk_command", required=True) - asub.add_parser("latest") - ad = asub.add_parser("download"); ad.add_argument("destination"); ad.add_argument("--url"); ad.add_argument("--sha256") - ai = asub.add_parser("inspect"); ai.add_argument("apk") - rp = sub.add_parser( - "player-render-pipeline", - help="extract the player-shipped MYSEKAI renderer pipeline and feature settings", - ) - rp.add_argument( - "--player-data", required=True, - help="extracted assets/bin/Data root containing globalgamemanagers", - ) - rp.add_argument( - "--types", required=True, - help="decoded managed assemblies for the same player build", - ) - rp.add_argument( - "--asset-pack", required=True, - help="APK/zip containing serialized resource files referenced by ResourceManager", - ) - rp.add_argument("--game-version", help="caller-known game version recorded as provenance") - rp.add_argument("--out", required=True, help="output JSON evidence path") - fal = sub.add_parser("fixture-actors", help="export all source-qualified character furniture animations") - fal.add_argument("--assets", required=True); fal.add_argument("--master", required=True) - fal.add_argument("--bundles", required=True); fal.add_argument("--workers", type=int, default=2) - fal.add_argument("--plan", action="store_true", help="write and summarize the source dependency plan without exporting") - fal.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="rotation key storage (default: snorm16); float32 disables lossy encoding") - fal.add_argument("--relation", action="append", choices=("fixture", "cut_scene"), default=None, - help="export only these relations' libraries (repeatable; default: all)") - ptc = sub.add_parser("player-timeline-clips", - help="export each player fixture timeline's CharacterAnimator clips against the " - "player avatar rig, one clip file per timeline package") - ptc.add_argument("--assets", required=True, - help="asset root holding fixture-timeline/ and the player timeline rows") - ptc.add_argument("--bundles", required=True, - help="directory of decrypted bundles (the avatar model and the fixture models)") - ptc.add_argument("--plan", action="store_true", help="summarize the clip plan without exporting") - stl = sub.add_parser("site-timelines", - help="export a site package's timeline tables, its step items (each prefab its " - "own glb with its prop clips) and its scene directors' rigs") - stl.add_argument("--assets", required=True, help="asset root receiving site-timeline/") - stl.add_argument("--bundles", required=True, - help="directory of decrypted bundles (the site packages and their dependencies)") - stl.add_argument("--package", action="append", required=True, - help="site package to export, e.g. mysekai__site__field__festivalgarden; repeatable") - fc = sub.add_parser("fixture-coverage", help="join every fixture master ID to extracted coverage") - fc.add_argument("--data-root", default=DEFAULT_DATA_ROOT); fc.add_argument("--out") - fc.add_argument("--region"); fc.add_argument("--source-version"); fc.add_argument("--json", action="store_true") - jp = sub.add_parser("jp", help="discover and download Japanese-server original bundles") - jsub = jp.add_subparsers(dest="jp_command", required=True) - jd = jsub.add_parser("discover", help="compare bootstrap metadata with the live official version list") - jd.add_argument("--bootstrap-url", default=None); jd.add_argument("--timeout", type=float, default=30.0) - jd.add_argument("--retries", type=int, default=3) - jm = jsub.add_parser("manifest", help="create a versioned JP bundle descriptor") - jm.add_argument("--app-version"); jm.add_argument("--app-hash") - jm.add_argument("--asset-version"); jm.add_argument("--asset-hash") - jm.add_argument("--versions-url", default=None) - jm.add_argument("--guest", action="store_true", help="explicitly create a guest only when public asset identity is unavailable") - jm.add_argument("--session", help="private guest-session JSON path; required with --guest") - jm.add_argument("--out", required=True); jm.add_argument("--timeout", type=float, default=30.0) - jm.add_argument("--retries", type=int, default=3) - jm.add_argument("--json", action="store_true", help="print the full descriptor as well as writing --out") - jpl = jsub.add_parser("plan", help="dependency-closed dry run from an online or offline descriptor") - jpl.add_argument("--manifest", required=True); jpl.add_argument("--family", action="append", default=[]) - jpl.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") - jpl.add_argument("--json", action="store_true", help="print root and bundle lists, not only counts") - jrt = jsub.add_parser("prepare", help="assemble a region-qualified Moly runtime in a separate staging directory") - jrt.add_argument("--manifest", required=True); jrt.add_argument("--workspace", required=True) - jrt.add_argument("--out", required=True); jrt.add_argument("--ui-from", required=True) - jrt.add_argument("--ui-jp-from", required=True, - help="JP UI extraction (ui-layout --region jp); its documents replace their donor copies") - jrt.add_argument("--vgmstream"); jrt.add_argument("--ffmpeg") - jrt.add_argument("--master-from", help="completed source-pinned master directory; defaults to Haruki when available") - jrt.add_argument("--actor-workers", type=int, default=2) - jrt.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") - jms = jsub.add_parser("master", help="download an immutable, bundle-matched JP master snapshot") - jms.add_argument("--manifest", required=True); jms.add_argument("--out", required=True) - jms.add_argument("--revision", default="main"); jms.add_argument("--workers", type=int, default=6) - jms.add_argument("--provider", choices=("haruki", "sekai-world"), default="haruki") - jms.add_argument("--all-tables", action="store_true", help="fetch all master tables, not only Moly dependencies") - jdl = jsub.add_parser("download", help="download and verify original JP bundles without publishing an extracted release") - jdl.add_argument("--manifest", required=True); jdl.add_argument("--out") - jdl.add_argument("--workers", type=int, default=8); jdl.add_argument("--retries", type=int, default=4) - jdl.add_argument("--root", action="append", default=[]) - jdl.add_argument("--family", action="append", default=[]) - jpu = jsub.add_parser("pull", help="download, decrypt and extract a JP descriptor") - jpu.add_argument("--manifest", required=True); jpu.add_argument("--out") - jpu.add_argument("--extract-out", default="local-data-jp") - jpu.add_argument("--workers", type=int, default=4); jpu.add_argument("--retries", type=int, default=4) - jpu.add_argument("--master"); jpu.add_argument("--master-cache") - jpu.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") - s = sub.add_parser("shader", help="what a package's shaders declare and contain") - s.add_argument("package", help="one AssetBundle file") - s.add_argument("--platform", type=int, default=None, - help="count only this ShaderCompilerPlatform (9 is GLSL text)") - s.add_argument("--out", help="write the full census here as JSON") - s.add_argument("--programs", metavar="DIR", - help="write each program body here, named by shader, platform and record") - s.add_argument("--json", action="store_true", help="print the counts as JSON") - - args = ap.parse_args(argv); _configure_unity(args.unity_version) - if args.cmd == "fixture-actors": - from chara.fixture_actor_libraries import build_plan, export_fixture_actor_libraries - if args.plan: - report = build_plan(args.assets, args.master, args.bundles, rotation_encoding=args.rotation_encoding) - print(json.dumps({"summary": report["summary"], "gaps": report["gaps"]}, ensure_ascii=False)) - return 1 if report["gaps"] else 0 - report = export_fixture_actor_libraries(args.assets, args.master, args.bundles, - workers=args.workers, unity_version=args.unity_version or "2022.3.62f3", - rotation_encoding=args.rotation_encoding, relations=args.relation, - progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) - print(json.dumps({key: value for key, value in report.items() if key != "results"}, ensure_ascii=False)) - return 1 if report["failed"] else 0 - if args.cmd == "player-timeline-clips": - from chara.player_timeline_clips import build_plan, export_player_timeline_clips - if args.plan: - plan = build_plan(args.assets) - print(json.dumps({"summary": plan["summary"], "gaps": plan["gaps"]}, ensure_ascii=False)) - return 1 if plan["gaps"] else 0 - report = export_player_timeline_clips(args.assets, args.bundles, - progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) - print(json.dumps({key: value for key, value in report.items() if key != "packages"}, - ensure_ascii=False)) - return 1 if report["gaps"] else 0 - if args.cmd == "site-timelines": - from sites.step_items import export_site_timelines - report = export_site_timelines(args.bundles, args.assets, list(args.package)) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "fixture-coverage": - from fixtures.coverage import write_coverage - path, report = write_coverage(args.data_root, args.out, region=args.region, - source_version=args.source_version) - print(json.dumps(report if args.json else {**report["summary"], "report": str(path)}, ensure_ascii=False)) - return 1 if report["summary"]["runtime"]["failed"] else 0 - if args.cmd == "jp": - from . import jp as jp_assets - try: - if args.jp_command == "discover": - options = {"timeout": args.timeout, "retries": args.retries} - if args.bootstrap_url: options["bootstrap_url"] = args.bootstrap_url - report = jp_assets.discover(**options) - elif args.jp_command == "manifest": - report = jp_assets.fetch_manifest(args.out, app_version=args.app_version, - app_hash=args.app_hash, session_file=args.session, - asset_version=args.asset_version, - asset_hash=args.asset_hash, guest=args.guest, - **({"versions_url": args.versions_url} - if args.versions_url else {}), - timeout=args.timeout, retries=args.retries) - elif args.jp_command == "plan": - report = jp_assets.plan(args.manifest, families=args.family, include=args.root) - elif args.jp_command == "prepare": - from .runtime_pack import prepare_runtime - report = prepare_runtime(args.manifest, args.workspace, args.out, ui_from=args.ui_from, - ui_jp_from=args.ui_jp_from, - decoder=args.vgmstream, transcoder=args.ffmpeg, - master_from=args.master_from, actor_workers=args.actor_workers, - rotation_encoding=args.rotation_encoding, - progress=lambda label: print(label, file=sys.stderr)) - elif args.jp_command == "master": - from .jp_master import download_master_snapshot - report = download_master_snapshot(args.manifest, args.out, revision=args.revision, workers=args.workers, provider=args.provider, all_tables=args.all_tables) - report = {"identity": report["identity"], "tableCount": report["tableCount"], "out": args.out} - elif args.jp_command == "download": - from .jp_download import download_descriptor - report = download_descriptor(args.manifest, out=args.out, workers=args.workers, - retries=args.retries, include=args.root, families=args.family, - progress=lambda row: print(f"{row['status']}: {row['completed']}/{row['requested']} verified, {row['failed']} failed", file=sys.stderr)) - else: - report = jp_assets.pull_descriptor(args.manifest, out=args.out, - extract_out=args.extract_out, workers=args.workers, - retries=args.retries, master=args.master, - master_cache=args.master_cache, - include=args.root) - if args.jp_command == "manifest" and not args.json: - shown = {"source": report["source"], "bundles": len(report["bundles"]), "descriptor": args.out} - elif args.jp_command == "plan" and not args.json: - shown = {"source": report["source"], **report["summary"]} - else: - shown = report - print(json.dumps(shown, ensure_ascii=False)); return 0 - except (jp_assets.JPError, OSError, ValueError, KeyError) as exc: - print(f"error: {exc}", file=sys.stderr); return 1 - if args.cmd == "pull": - from .fetch import pull - source, cache = _master_source(args) - report = pull(args.manifest, args.out, args.asset_base_url, args.workers, args.retries, - master=source, master_cache=cache, extract_out=args.extract_out, - vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) - if args.json: - print(json.dumps(report, ensure_ascii=False)) - else: - print(f"downloaded {report['downloads']}/{report['requiredBundles']} bundles") - # The sound packages are the one part of the set that master rows name, - # so "no audio" has a reason the caller must see without asking for JSON. - audio = report.get("audio") or {} - if audio.get("roots"): - print(f"audio: {len(audio['roots'])} sound packages named by master tables") - elif audio.get("error"): - print(f"audio: no sound package pulled ({audio['error']})") - if audio.get("notInManifest"): - print(f"audio: {len(audio['notInManifest'])} named sound packages " - f"are not in this manifest: " - f"{', '.join(audio['notInManifest'])}") - _print_extract_summary(report["extraction"], args.extract_out or DEFAULT_DATA_ROOT) - return 1 if _extract_failed(report["extraction"]) else 0 - if args.cmd == "shader": - import UnityPy - from shaders import census as shader_census - entries = shader_census.census(UnityPy.load(args.package), - keep_code=bool(args.programs)) - counts = shader_census.totals(entries, args.platform) - if args.out: - from .jsonio import write_json - write_json(args.out, {"package": args.package, - "shaders": shader_census.without_code(entries)}) - if args.programs: - written = _write_programs(args.programs, entries, args.platform) - counts["bodiesWritten"] = written - if args.json: - print(json.dumps(counts, ensure_ascii=False)) - else: - scope = "all platforms" if args.platform is None else f"platform {args.platform}" - print(f"{counts['shaders']} shaders, {counts['records']} program records, " - f"{counts['uniquePrograms']} distinct programs ({scope})") - if counts["platformErrors"]: - print(f"{counts['platformErrors']} platform blob(s) failed to parse") - for name, records, unique in shader_census.by_shader(entries, args.platform)[:10]: - print(f" {unique:6d} distinct / {records:6d} records {name}") - return 1 if counts["platformErrors"] else 0 - if args.cmd == "ui-audio": - from ui.audio import extract_ui_audio - report = extract_ui_audio(args.apk, args.out, decoder=args.vgmstream, transcoder=args.ffmpeg) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "fetch-apk": - from . import apk - forwarded = [] - if args.endpoint: forwarded += ["--endpoint", args.endpoint] - forwarded += ["--timeout", str(args.timeout), "--retries", str(args.retries), args.apk_command] - if args.apk_command == "download": - forwarded += [args.destination] - if args.url: forwarded += ["--url", args.url] - if args.sha256: forwarded += ["--sha256", args.sha256] - elif args.apk_command == "inspect": - forwarded += [args.apk] - return apk.main(forwarded) - if args.cmd == "player-render-pipeline": - from .player_render_pipeline import ( - extract_mysekai_render_pipeline, - write_mysekai_render_pipeline, - ) - try: - document = extract_mysekai_render_pipeline( - args.player_data, - args.types, - args.asset_pack, - unity_version=args.unity_version, - game_version=args.game_version, - ) - write_mysekai_render_pipeline(args.out, document) - except (OSError, ValueError, KeyError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - first = document["rendererSlots"][0]["objectKey"] if document["rendererSlots"] else None - count = len(document["renderers"].get(first, {}).get("features", [])) if first else 0 - print(json.dumps({ - "unityVersion": document["unityVersion"], - "gameVersion": document["gameVersion"], - "rendererSlots": len(document["rendererSlots"]), - "primaryRendererFeatures": count, - "out": args.out, - }, ensure_ascii=False)) - return 0 - if args.cmd == "extract": - from .extract import extract_manifest - from .assets.packages import builtin_archive_paths - source, cache = _master_source(args) - report = extract_manifest(args.manifest, args.bundles, args.out, args.unity_version, - master=source, master_cache=cache, - player_data=args.player_data, - player_types=args.player_types, - player_declarations=args.player_declarations, - fixture_meshes=args.fixture_meshes, - fixture_particles=args.fixture_particles, - rotation_encoding=args.rotation_encoding, - shader_compiler=args.shader_compiler, - builtin_resources=builtin_archive_paths( - args.builtin_resources), - vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) - if args.json: - print(json.dumps(report, ensure_ascii=False)) - else: - if report.get("discovery"): - d = report["discovery"] - print(f"discovered {d['selected']} pack bundles under {args.bundles} " - f"(ignored {d['ignored']} unrelated files)") - _print_extract_summary(report, args.out) - return 1 if _extract_failed(report) else 0 - from chara import characters - if args.cmd == "motion-library": - from chara.motion_library import export_motion_library - names = [s for s in args.clips.split(",") if s] if args.clips else None - report = export_motion_library(args.reference_bundle, args.motion_bundle, args.out_dir, args.name, - aux=tuple(args.aux_bundle), names=names, - rotation_encoding=args.rotation_encoding, - controller_bundles=args.controller_bundle) - print(json.dumps(report, ensure_ascii=False)); return 0 - if args.cmd == "registry": - source, cache = _master_source(args) - if not source: - ap.error("registry needs master tables: pass --master or --master-url") - from .assets.manifest import parse_manifest - from .assets.router import route - from .extract import _unit_id - from .master import Master, MissingTable - from chara.registry import build_registry - names = parse_manifest(args.manifest) - units = sorted({u for n in names if route(n) and route(n).domain == "character" - for u in [_unit_id(n)] if u is not None}) - index = None - if args.motion_index: - with open(args.motion_index, encoding="utf-8") as handle: - index = json.load(handle) - source, cache = _master_source(args) - master = Master(source, cache_dir=cache) - try: - client_configs = master.client_configs() - except MissingTable: - client_configs = None - document = build_registry(master, units, index, client_configs=client_configs) - with open(args.out, "w", encoding="utf-8", newline="\n") as handle: - json.dump(document, handle, ensure_ascii=False, indent=1, allow_nan=False) - handle.write("\n") - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - - if args.cmd == "talks": - source, cache = _master_source(args) - if not source: - ap.error("talks needs master tables: pass --master or --master-url") - from chara.talks import extract_talks - source, cache = _master_source(args) - print(json.dumps(extract_talks(source, args.bundle, args.out, - master_cache=cache, - lib_bundle=args.lib), ensure_ascii=False)) - return 0 - - if args.cmd == "tweets": - source, cache = _master_source(args) - if not source: - ap.error("tweets needs master tables: pass --master or --master-url") - from chara.tweets import extract_tweets - print(json.dumps(extract_tweets(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "tweet-tables": - source, cache = _master_source(args) - if not source: - ap.error("tweet-tables needs master tables: " - "pass --master or --master-url") - from chara.tweet_tables import extract_tweet_tables - print(json.dumps(extract_tweet_tables(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "site-groups": - source, cache = _master_source(args) - if not source: - ap.error("site-groups needs master tables: " - "pass --master or --master-url") - from sites.site_groups import extract_site_groups - print(json.dumps(extract_site_groups(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "ui-action-icons": - from ui.action_icon import export_action_icons - print(json.dumps(export_action_icons(args.bundle, args.out_dir), - ensure_ascii=False)) - return 0 - - if args.cmd == "browser-assets": - from ui.character_portrait import export_character_portraits - result = export_character_portraits(args.player_data, args.asset_root, - source_region=args.source_region, - source_game_version=args.source_game_version) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 - - if args.cmd == "fixture-presentation": - from fixtures.presentation import extract_custom_geometry, enrich_presentation - extracted = extract_custom_geometry(args.bundles, Path(args.asset_root) / "custom-fixture-models") - if extracted.get("failed"): - print(json.dumps({"customGeometry": extracted}, ensure_ascii=False)) - return 1 - print(json.dumps(enrich_presentation(args.master, args.asset_root), ensure_ascii=False)) - return 0 - - if args.cmd == "validate-runtime": - from .runtime_validation import validate_runtime - report = validate_runtime(args.asset_root, verify_actor_hashes=args.verify_actor_hashes) - print(json.dumps(report, ensure_ascii=False)) - return 1 if report["missingResources"] else 0 - - if args.cmd == "fixture-animations": - from fixtures.animation_refresh import plan_animation_refresh, refresh_fixture_animations - report = plan_animation_refresh(args.asset_root) if args.plan else refresh_fixture_animations( - args.asset_root, args.bundles, packages=args.package, - progress=lambda row: print(json.dumps(row), file=sys.stderr)) - print(json.dumps(report, ensure_ascii=False)) - return 0 - - if args.cmd == "timeline-capabilities": - from fixtures.timeline_capabilities import build_capabilities - report = build_capabilities(args.asset_root) - print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) - return 0 - - if args.cmd == "ui-layout" and args.region == "jp": - from ui.prefab_layout import extract_layout_jp - if not (args.apk and args.client_version and args.master and args.prefab): - ap.error("ui-layout --region jp needs --apk (base and asset pack), " - "--client-version, --master and at least one --prefab") - result = extract_layout_jp(args.apk, args.out_dir, client_version=args.client_version, - master_dir=args.master, prefabs=args.prefab, - bundles_root=args.bundles, - font_bundles_root=args.font_bundles) - for item in result["written"]: - summary = item["summary"] - print(f"layout: {item['family']}/{item['prefab']} " - f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " - f"texts={summary['textComponents']} " - f"partial={len(summary['partialComponents'])} " - f"unresolved={len(summary['unresolvedPointers'])}") - for item in result["failures"]: - print(f"layout FAILED {item['prefab']}: {item['reason']}") - print(f"manifest: {result['manifestPath']}") - return 0 if not result["failures"] else 1 - - if args.cmd == "ui-layout": - from ui.prefab_layout import extract_layout - if not args.player_data: - ap.error("ui-layout --region cn needs --player-data") - result = extract_layout(args.player_data, args.out_dir, - bundles_root=args.bundles, - bundle_manifest=args.bundle_manifest, - prefabs=args.prefab, - client_version=args.client_version) - for item in result["settings"]: - print(f"settings: {item['document']} from {item['serializedFile']}:{item['pathId']}") - if not result["settings"]: - print("settings: palette.json and tween-settings.json not written " - "(they name the client version: pass --client-version)") - for item in result["written"]: - summary = item["summary"] - print(f"layout: {item['family']}/{item['prefab']} " - f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " - f"texts={summary['textComponents']} " - f"partial={len(summary['partialComponents'])}") - for item in result["failures"]: - print(f"layout FAILED {item['prefab']}: {item['reason']}") - print(f"census: {result['censusPath']}") - return 0 if not result["failures"] else 1 - - if args.cmd == "ui-wordings": - from ui.prefab_layout import extract_container_wordings - result = extract_container_wordings(args.player_data, args.master, args.out, args.client_version) - print(f"wordings: {result['rows']} entries ({result['builtinRows']} built-in, " - f"{result['masterRows']} master rows, {result['masterOverwroteBuiltin']} built-in " - f"overwritten, {result['masterValuesRewritten']} master values rewritten, " - f"{result['builtinRejectedLines']} rejected lines) from " - f"{result['serializedFile']}:{result['pathId']} -> {result['path']}") - return 0 - - if args.cmd == "fixture-reactions": - from fixtures.reactions import extract_fixture_reactions - print(json.dumps(extract_fixture_reactions(args.bundle, args.out), ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-end-motions": - from sites.birthday_end_motions import extract_birthday_end_motions - print(json.dumps(extract_birthday_end_motions(args.bundle, args.out), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-party-talks": - source, cache = _master_source(args) - if not source: - ap.error("birthday-party-talks needs master tables: " - "pass --master or --master-url") - from chara.talks import attach_birthday_talks - print(json.dumps(attach_birthday_talks(source, args.bundle, args.talks, - args.out, master_cache=cache, - lib_bundle=args.lib), - ensure_ascii=False)) - return 0 - - if args.cmd == "fixture-master-slice": - source, cache = _master_source(args) - if not source: - ap.error("fixture-master-slice needs master tables: " - "pass --master or --master-url") - from fixtures.master_slice import export_fixture_master_slice - print(json.dumps(export_fixture_master_slice(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-parties": - source, cache = _master_source(args) - if not source: - ap.error("birthday-parties needs master tables: " - "pass --master or --master-url") - from sites.birthday_parties import extract_birthday_parties - print(json.dumps(extract_birthday_parties(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-party-delivery": - source, cache = _master_source(args) - if not source: - ap.error("birthday-party-delivery needs master tables: " - "pass --master or --master-url") - from sites.birthday_party_delivery import extract_birthday_party_delivery - print(json.dumps(extract_birthday_party_delivery(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "gate-visit-policy": - source, cache = _master_source(args) - if not source: - ap.error("gate-visit-policy needs master tables: " - "pass --master or --master-url") - from sites.gate_visit_policy import extract_gate_visit_policy - print(json.dumps(extract_gate_visit_policy(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - if args.cmd == "time-manager": - from core.time_manager import extract_time_manager - document = extract_time_manager(args.source, args.out) - print(json.dumps({"source": document["source"], "timeManager": document["timeManager"]}, - ensure_ascii=False)) - return 0 - if args.cmd == "client-config": - source, cache = _master_source(args) - if not source: - ap.error("client-config needs master tables: " - "pass --master or --master-url") - from core.client_config import extract_client_config - print(json.dumps(extract_client_config(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "master-tables": - source, cache = _master_source(args) - if not source: - ap.error("master-tables needs master tables: " - "pass --master or --master-url") - from core.master_tables import extract_master_tables - print(json.dumps(extract_master_tables(source, args.out, - master_cache=cache, tables=args.tables), - ensure_ascii=False)) - return 0 - - if args.cmd == "emoticons": - from chara.emoticons import extract_emoticons - print(json.dumps(extract_emoticons(args.bundle, args.out_dir), ensure_ascii=False)) - return 0 - if args.cmd == "avatar-parts": - from chara.avatar_parts import extract_avatar_parts - print(json.dumps(extract_avatar_parts(args.bundle, args.out_dir), ensure_ascii=False)) - return 0 - if args.cmd == "harvest-tools": - from chara.harvest_tools import extract_harvest_tools - result = extract_harvest_tools(args.bundle, args.out_dir, bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "player-tool-models": - from chara.harvest_tools import extract_player_tool_models - result = extract_player_tool_models(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "effect-resources": - from .resource_exports import export_effect_resources - print(json.dumps(export_effect_resources( - args.player_data, args.types, args.out, declarations=args.declarations, - bundle_root=args.bundle_root), ensure_ascii=False)) - return 0 - if args.cmd == "layout-grid": - from .layout_grid import export_layout_grid - print(json.dumps(export_layout_grid(args.player_data, args.types, args.out_dir), - ensure_ascii=False, indent=2)) - return 0 - if args.cmd == "drop-item-shade": - from .resource_exports import export_drop_item_shade - print(json.dumps(export_drop_item_shade(args.player_data, args.types, args.out), - ensure_ascii=False)) - return 0 - if args.cmd == "stencil-shadow": - from .resource_exports import export_stencil_shadow - print(json.dumps(export_stencil_shadow(args.player_data, args.types, args.out), - ensure_ascii=False)) - return 0 - if args.cmd == "resource-prefab": - from .resource_exports import export_resource_prefab - from .assets.packages import builtin_archive_paths - document = export_resource_prefab(args.player_data, args.types, - args.load_name, args.out_dir, - builtin_archive_paths(args.builtin_resources)) - print(json.dumps({"file": document["file"], "objects": document["objects"], - "roots": [root["name"] for root in document["roots"]], - "particles": len(document["particles"]), - "unsupported": len(document["unsupported"])}, - ensure_ascii=False)) - return 0 - if args.cmd == "resource-programs": - from .resource_exports import export_resource_programs - document = export_resource_programs(args.player_data, args.types, args.load_name, - args.out_dir, args.shader_compiler) - print(json.dumps({"file": document["file"], - "shaders": [(row["name"], row["state"], len(row["materials"])) - for row in document["shaders"]]}, ensure_ascii=False)) - return 0 - if args.cmd == "fixture-particles": - from .assets.packages import PackageStore, builtin_archive_paths, require_paths - from fixtures.particles import DOMAINS, extract_from_store - require_paths(args.bundle) - builtins = builtin_archive_paths(args.builtin_resources) - require_paths(builtins, "built-in resource") - store = PackageStore(list(args.bundle) + builtins, args.bundle_root) - result = extract_from_store(store, args.out_dir, - shader_compiler=args.shader_compiler, - domains=tuple(args.domain or DOMAINS)) - print(json.dumps(result, ensure_ascii=False)) - return 0 - if args.cmd == "prefab-package": - from sites.prefab_package import extract_prefab_package - print(json.dumps(extract_prefab_package(args.bundle, args.prefab, args.out_dir, - bundle_root=args.bundle_root), - ensure_ascii=False)) - return 0 - if args.cmd == "site-sound-wave": - from sites.sound_wave import extract_site_sound_wave - result = extract_site_sound_wave(args.bundle, args.out, bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if result["failed"] else 0 - if args.cmd == "site-common-action-effects": - from sites.action_effects import extract_site_common_action_effects - result = extract_site_common_action_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-harvest-object-effects": - from sites.action_effects import extract_site_harvest_object_effects - result = extract_site_harvest_object_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-action-effects": - from sites.action_effects import extract_site_action_effects - result = extract_site_action_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-home-effects": - from sites.action_effects import extract_site_home_effects - result = extract_site_home_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "fixture-put-effects": - from sites.action_effects import extract_fixture_put_effects - result = extract_fixture_put_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "player-avatar": - from chara.player_avatar import export_player_avatar, write_motion_manifest - record = export_player_avatar(args.bundle, args.out_dir, name=args.name) - manifest_path = str(Path(args.out_dir) / f"{args.name}.motion-manifest.json") - write_motion_manifest(record, manifest_path) - print(json.dumps({ - "glb": record["glb"], "index": record["index"], - "motionManifest": manifest_path, - "sourcePackages": record["sourcePackages"], - "playerMesh": record["playerMesh"], - "counts": record["counts"]}, ensure_ascii=False)) - return 0 - if args.cmd == "phenomena": - from .assets.packages import builtin_archive_paths, require_paths - from phenomena.environments import extract_phenomena - source, cache = _master_source(args) - # --bundle takes paths, not bare package names: PackageStore keys the - # store by basename and UnityPy.load answers a path that is not there - # with an empty environment and no error, so a bare name runs green - # over zero objects. Refuse it here, before any work is done. - require_paths(args.bundle) - require_paths(args.site_bundle) - report = extract_phenomena(args.bundle, args.out_dir, - bundle_root=args.bundle_root, master=source, - master_cache=cache, vgmstream=args.vgmstream, - ffmpeg=args.ffmpeg, - shader_compiler=args.shader_compiler, - extra_archives=builtin_archive_paths( - args.builtin_resources), - site_bundles=args.site_bundle) - print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, - ensure_ascii=False)) - return 0 - if args.cmd == "mysekai-audio": - from phenomena.audio_corpus import extract_audio_corpus - report = extract_audio_corpus(args.talks, args.manifest, - args.bundle_root, args.out_dir, - decoder=args.vgmstream, - transcoder=args.ffmpeg) - print(json.dumps({k: v for k, v in report.items() if k != "audio"}, - ensure_ascii=False)) - return 0 - if args.cmd == "cutscene-audio": - from phenomena.cutscene_audio import extract_cutscene_audio - report = extract_cutscene_audio(args.clips, args.cut_scenes, args.manifest, - args.bundle_root, args.out_dir, - decoder=args.vgmstream, transcoder=args.ffmpeg) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "audio-sequences": - if Path(args.out).resolve() == Path(args.loop).resolve(): - ap.error("audio-sequences writes a copy: --out must differ from --loop") - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.audio_sequences import annotate_loop - report = annotate_loop(args.loop, args.bundle_root, args.out) - print(json.dumps({"counts": report["counts"], - "structuredByType": report["structuredByType"], - "archiveCuesByReferenceType": report["archiveCuesByReferenceType"], - "archiveSequencesByType": report["archiveSequencesByType"], - "answeredByTree": {k: len(v) for k, v - in report["answeredByTree"].items()}, - "failures": len(report["failures"]), - "disagreements": len(report["disagreements"]), - "sampleDifferences": len(report["sampleDifferences"])}, - ensure_ascii=False)) - return 0 - if args.cmd == "record-audio": - source, cache = _master_source(args) - if not source: - ap.error("record-audio needs master tables: " - "pass --master or --master-url") - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.record_audio import extract_record_audio - document = extract_record_audio(source, args.bundle_root, args.out, - bundle_version=args.bundle_version, - decoder=args.vgmstream, master_cache=cache, - only=args.only) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-storage": - from phenomena.record_audio import attach_storage - document = attach_storage(args.input, args.out, server=args.server, - workers=args.workers) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-carry": - source, cache = _master_source(args) - if not source: - ap.error("record-audio-carry needs the target master tables: " - "pass --master or --master-url") - from phenomena.record_audio import carry_record_audio - document = carry_record_audio(source, args.manifest, args.source, - args.source_manifest, args.out, - region=args.region, master_cache=cache) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-fetch": - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.record_audio import fetch_record_bundles - receipt = fetch_record_bundles(args.manifest, args.document, args.asset_base_url, - args.out, workers=args.workers) - print(json.dumps(receipt["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-merge": - from phenomena.record_audio import merge_record_audio - print(json.dumps(merge_record_audio(args.inputs, args.out)["summary"], - ensure_ascii=False)) - return 0 - if args.cmd == "audio-categories": - from phenomena.audio_categories import write_category_table - document = write_category_table(args.configuration, args.out) - print(json.dumps({"source": document["source"], - "categories": [(row["name"], row["index"], row["volume"]) - for row in document["categories"]]}, - ensure_ascii=False)) - return 0 - if args.cmd == "partvoice-routes": - source, cache = _master_source(args) - if not source: - ap.error("partvoice-routes needs master tables: " - "pass --master or --master-url") - from .master import Master - from phenomena.partvoice import (build_partvoice_routes, - route_counts, write_partvoice_routes) - roster = json.loads(Path(args.roster).read_text(encoding="utf-8")) - document = build_partvoice_routes(Master(source, cache_dir=cache), - roster) - path = write_partvoice_routes(document, args.out_dir) - print(json.dumps({**route_counts(document), "file": str(path)}, - ensure_ascii=False)) - return 0 - if args.cmd == "material-thumbnails": - from ui.material_thumbnail import export_material_thumbnails - from .extract import DEFAULT_UNITY_VERSION - result = export_material_thumbnails( - args.bundle, Path(args.master) / "materials.json", args.out_dir, - unity_version=args.unity_version or DEFAULT_UNITY_VERSION) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 if not result["coverage"]["missingMaterialIds"] else 1 - if args.cmd == "honor-images": - from ui.honor_images import export_honor_images - from .extract import DEFAULT_UNITY_VERSION - result = export_honor_images(args.master, args.bundles, args.out_dir, - unity_version=args.unity_version or DEFAULT_UNITY_VERSION) - print(json.dumps({"counts": result["counts"], "bundles": len(result["bundles"]), - "missingBundles": result["missingBundles"]}, ensure_ascii=False)) - return 0 if not result["missingBundles"] else 1 - if args.cmd == "item-icons": - from ui.item_icon import export_item_icons, package_paths - packages = package_paths(args.decrypted) - masters = {} - if args.root: - for key, name in (("materials", "mysekai-materials.json"), - ("items", "mysekai-items.json"), - ("fixtures", "mysekai-fixtures.json"), - ("tools", "mysekai-tools.json")): - path = Path(args.root) / name - masters[key] = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None - result = export_item_icons(packages, args.out_dir, **masters) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 - if args.cmd == "sitemap-screen-layer": - from sites.screen_layer import extract_screen_layer_jp - result = extract_screen_layer_jp( - args.apk, Path(args.out_dir) / "site" / "sitemap" / "screen_layer", - client_version=args.client_version) - print(json.dumps({"counts": result["counts"], "path": result["path"]}, - ensure_ascii=False)) - return 0 - if args.cmd == "site": - from .assets.packages import require_paths - from sites.pack import extract_sites - source, cache = _master_source(args) - # Same refusal as the phenomena command: a bare package name under - # --bundle would extract zero objects and report green. - require_paths(args.bundle) - report = extract_sites(args.bundle, args.out_dir, - bundle_root=args.bundle_root, master=source, - master_cache=cache) - print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, - ensure_ascii=False)) - return 0 - if args.cmd == "alone-actions": - from chara.alone_actions import write_alone_actions - print(json.dumps(write_alone_actions(args.bundle, args.out), ensure_ascii=False)) - return 0 - if args.cmd == "facial-tables": - tables = characters.facial_tables(args.bundle) - with open(args.out, "w", encoding="utf-8", newline="\n") as fh: - json.dump(tables, fh, ensure_ascii=False, indent=1, allow_nan=False); fh.write("\n") - print(json.dumps({k: len(v) for k, v in tables.items()})); return 0 - sampled = None - if args.motion_bundle and args.clips: - sampled = characters.sample_clips(args.motion_bundle, [s for s in args.clips.split(",") if s]) - facial = None - if args.facial_tables: - with open(args.facial_tables, encoding="utf-8") as fh: facial = json.load(fh) - cw, ch = args.atlas_cell.lower().split("x") - report = characters.extract_character(args.bundle, args.out_dir, args.name, sampled=sampled, unit_id=args.unit_id, facial=facial, atlas_cell=(int(cw), int(ch)), with_cloth=not args.no_cloth, aux=tuple(args.aux_bundle)) - print(json.dumps(report, ensure_ascii=False)); return 0 - +from cli.main import main +from cli.legacy import _extract_failed, _print_extract_summary if __name__ == "__main__": sys.exit(main()) diff --git a/src/core/content.py b/src/core/content.py new file mode 100644 index 0000000..0dd5ad3 --- /dev/null +++ b/src/core/content.py @@ -0,0 +1,20 @@ +"""Canonical bytes and hashes for content-addressed contract documents. + +This is the existing pack v2 encoding. Domain writers keep their authored +nonfinite-number representation; this encoder accepts only finite JSON. +""" +import hashlib +import json + + +def canonical_bytes(document): + return (json.dumps(document, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False) + "\n").encode("utf-8") + + +def content_hash(data): + return hashlib.sha256(data).hexdigest() + + +def document_hash(document): + return content_hash(canonical_bytes(document)) diff --git a/src/core/extract.py b/src/core/extract.py index 58f3bc3..d789ff4 100644 --- a/src/core/extract.py +++ b/src/core/extract.py @@ -122,6 +122,7 @@ "motion": {"AnimationClip", "Avatar"}, "facial": {"MonoBehaviour"}, "performance": {"TextAsset"}, + "communication": {"TextAsset"}, "talk": {"TextAsset"}, "talk-constants": {"TextAsset"}, "camera": {"MonoBehaviour", "PlayableDirector"}, @@ -303,6 +304,7 @@ def _configure_unity_version(unity_version): "motion-library.index.json", "facial-tables.json", "alone-actions.json", + "communication-actions.json", "talks.json", "tweets.json", "emoticons/emoticons.json", @@ -1688,6 +1690,11 @@ def extract_manifest(manifest, bundles, out, unity_version=None, master=None, elif target.domain == "performance": from chara.alone_actions import write_alone_actions result = write_alone_actions(str(bundle), str(out / "alone-actions.json")) + elif target.domain == "communication": + if not master: + raise ValueError("communication actions require master tables") + from chara.communication_actions import extract_communication_actions + result = extract_communication_actions(master, str(bundle), str(out / "communication-actions.json"), master_cache) else: from chara.characters import facial_tables target_path = out / "facial-tables.json" diff --git a/src/pack/gc.py b/src/pack/gc.py index fa57d24..16968c7 100644 --- a/src/pack/gc.py +++ b/src/pack/gc.py @@ -25,6 +25,21 @@ def catalog_garbage(output): """List unused blobs only after validating every retained release root.""" output = Path(output) + if not (output / "asset-packs.json").exists(): + from .task_results import retained_results + roots, blobs = retained_results(output) + if not roots: + raise ValueError("no release or extraction result roots; refusing unrooted collection") + on_disk = {p.relative_to(output / "blobs").as_posix(): p.stat().st_size + for p in (output / "blobs").rglob("*") if p.is_file()} + garbage = sorted(on_disk.keys() - blobs.keys()) + # Without a release selector, preserve manifests and channel/archive + # state. Such a store may be mid-migration rather than task-only. + if any((output / folder).exists() for folder in ("catalogs", "channels", "packages")): + raise ValueError("release objects exist without a default catalog; restore it before GC") + return {"delete": garbage, "delete_manifests": [], "delete_manifest_bytes": 0, + "channel_roots": {}, "roots": roots, "delete_bytes": sum(on_disk[p] for p in garbage), + "retained_catalogs": 0, "referenced_blobs": sorted(blobs)} errors, info = verify_catalog(output / "asset-packs.json", root=output) if errors: raise ValueError("cannot determine catalog garbage: " + "; ".join(errors)) diff --git a/src/pack/groups.py b/src/pack/groups.py index 963dbde..250c89d 100644 --- a/src/pack/groups.py +++ b/src/pack/groups.py @@ -29,7 +29,7 @@ # checks it against the paths a browser run requested. ROOT_DOCUMENTS = ( "manifest.json", "source.json", "characters.json", "motion-library.glb", "motion-library.index.json", - "alone-actions.json", "facial-tables.json", "client-config.json", "birthday-parties.json", + "alone-actions.json", "communication-actions.json", "facial-tables.json", "client-config.json", "birthday-parties.json", "birthday-party-delivery.json", "mysekai-blueprints.json", "mysekai-fixtures.json", "mysekai-items.json", "mysekai-fixture-main-genres.json", "mysekai-fixture-sub-genres.json", @@ -66,7 +66,7 @@ "effect-resources", "graphics-config", "layout-grid", "site-action-effects", "site-common-action-effects", "site-harvest-object-effects", "site-home-effects", "fixture-put-effects", "site-sound-wave", "site-timeline", - "honor", "honor_frame", "material-thumbnails", + "honor", "honor_frame", "material-thumbnails", "text", ) FORMATS = {".json", ".glb", ".gltf", ".png", ".jpg", ".jpeg", ".webp", ".ktx2", ".rgba8", ".ogg", ".wav", ".mp3", ".bin"} BUILD_EVIDENCE_FILES = {"export-report.json", "export-plan.json", "export-progress.json", diff --git a/src/pack/identity.py b/src/pack/identity.py index e298b69..0198fcf 100644 --- a/src/pack/identity.py +++ b/src/pack/identity.py @@ -7,7 +7,7 @@ strings, booleans and null (with arrays/objects); it never contains floats. """ from __future__ import annotations -import json +from core.content import canonical_bytes from .hashing import sha256_bytes MANIFEST_V2 = "moly-asset-manifest/2" @@ -17,11 +17,6 @@ MAX_SAFE_INTEGER = (1 << 53) - 1 -def canonical_bytes(document): - return (json.dumps(document, ensure_ascii=False, sort_keys=True, - separators=(",", ":"), allow_nan=False) + "\n").encode("utf-8") - - def content_projection(manifest): projection = {"schema": CONTENT_V1, "transforms": manifest["transforms"], "entries": [{key: entry[key] for key in ("path", "bytes", "content_sha256", "xf")} diff --git a/src/pack/task_results.py b/src/pack/task_results.py new file mode 100644 index 0000000..dc0ce7b --- /dev/null +++ b/src/pack/task_results.py @@ -0,0 +1,56 @@ +"""Extraction result roots in the existing pack store. + +The result is committed after its artifacts. Every retained result protects +its artifacts from release GC even before a product selects them. Results +hold no input paths, local clocks, credentials or mutable release labels. +""" +import re + +from core.content import canonical_bytes, content_hash, document_hash +from .paths import output_path +from .verify import strict_json + + +def read_result(root, key): + if not re.fullmatch(r"[0-9a-f]{64}", key): + raise ValueError("invalid task result key") + path = output_path(root, f"tasks/{key}.json") + if not path.exists(): + return None + raw = path.read_bytes() + result = strict_json(raw) + if (result.get("schema") != "moly-result/1" or result.get("key") != key + or document_hash(result["keyParts"]) != key or result.get("status") != "ok" + or canonical_bytes(result) != raw): + raise ValueError("invalid task result identity") + seen = set() + for artifact in result["artifacts"]: + logical = artifact["path"] + if (not isinstance(logical, str) or "\\" in logical or ":" in logical + or any(part in ("", ".", "..") for part in logical.split("/"))): + raise ValueError("invalid task artifact logical path") + output_path(root, logical) + if logical in seen: + raise ValueError("duplicate task artifact path") + seen.add(logical) + digest = artifact["sha256"] + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("invalid task artifact hash") + if artifact["blob"] != f"{digest[:2]}/{digest}.bin": + raise ValueError("invalid task artifact storage address") + data = output_path(root, "blobs/" + artifact["blob"]).read_bytes() + if type(artifact["size"]) is not int or len(data) != artifact["size"] or content_hash(data) != digest: + raise ValueError(f"corrupt task artifact: {logical}") + return result + + +def retained_results(root): + """Return verified roots and blobs; corrupt roots abort collection.""" + folder = output_path(root, "tasks") + roots, blobs = {}, {} + for path in sorted(folder.glob("*.json")): + result = read_result(root, path.stem) + roots[f"tasks/{path.name}"] = "extraction result" + for artifact in result["artifacts"]: + blobs[artifact["blob"]] = artifact["size"] + return roots, blobs diff --git a/src/pack/verify.py b/src/pack/verify.py index e935d7d..38d48d8 100644 --- a/src/pack/verify.py +++ b/src/pack/verify.py @@ -730,6 +730,16 @@ def include(path, label): default = root / "asset-packs.json" if default.is_file() and default.resolve() != catalog_path: include(default, "default") + # Extraction results and published catalogs share these blobs. A result + # must retain its referenced artifacts before a runtime release selects it. + from .task_results import retained_results + try: + task_roots, task_blobs = retained_results(root) + roots.update(task_roots) + referenced.update(task_blobs) + info["blob_sizes"].update(task_blobs) + except (OSError, ValueError, KeyError, TypeError) as exc: + errors.append(f"extraction results: {exc}") on_disk = {path.relative_to(root / "blobs").as_posix() for path in (root / "blobs").rglob("*") if path.is_file()} package_files = {path.relative_to(root).as_posix() diff --git a/src/perf/fixture_talks.py b/src/perf/fixture_talks.py index 4a55a33..b32c5a2 100644 --- a/src/perf/fixture_talks.py +++ b/src/perf/fixture_talks.py @@ -166,7 +166,7 @@ def classify_ops(text, script_name=None): def _text_assets(bundle): """``{TextAsset name: script text}``.""" out = {} - for obj in UnityPy.load(bundle).objects: + for obj in UnityPy.load(Path(bundle).read_bytes()).objects: if obj.type.name != "TextAsset": continue tree = obj.read_typetree() diff --git a/src/phenomena/audio_corpus.py b/src/phenomena/audio_corpus.py index 2316f31..c72af9b 100644 --- a/src/phenomena/audio_corpus.py +++ b/src/phenomena/audio_corpus.py @@ -122,6 +122,7 @@ def _corpus_talks(document): for unit in (document.get("units") or {}).values(): yield from unit.get("talks") or [] yield from document.get("talks") or [] + yield from (document.get("commonSomeCharacter") or {}).get("talks") or [] def voice_requests(document): diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..ab35289 --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Versioned extraction tasks, reuse, and product assembly over the pack store.""" diff --git a/src/pipeline/contract.py b/src/pipeline/contract.py new file mode 100644 index 0000000..46cdeb1 --- /dev/null +++ b/src/pipeline/contract.py @@ -0,0 +1,44 @@ +"""Task descriptions separate content-affecting facts from source locations.""" +import re + +from core.content import document_hash + + +def key_parts(task): + return {"stage": task["stage"], "subject": task["subject"], + "params": task["params"], "atoms": task["atoms"], "context": task["context"], + "inputs": [{k: item[k] for k in ("role", "sha256", "size")} + for item in sorted(task["inputs"], key=lambda item: item["role"])]} + + +def validate(task): + if task.get("schema") != "moly-task/1": + raise ValueError("unsupported task schema") + if (not isinstance(task["subject"], str) or not task["subject"] + or not re.fullmatch(r"[a-z][a-z0-9_.-]*", task["stage"]["name"]) + or type(task["stage"]["version"]) is not int or task["stage"]["version"] < 1): + raise ValueError("invalid task stage or subject") + roles = set() + for item in task["inputs"]: + role = item["role"] + if not isinstance(role, str) or not role or role in roles: + raise ValueError("empty or duplicate task input role") + roles.add(role) + if (not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) + or type(item["size"]) is not int or item["size"] < 0 + or not isinstance(item.get("locators"), list)): + raise ValueError("invalid input identity or locators") + for field in ("params", "atoms", "context"): + if not isinstance(task[field], dict): + raise ValueError(f"{field} must be an object") + if task.get("key") != document_hash(key_parts(task)): + raise ValueError("task key does not match its declared inputs") + return task + + +def describe(stage, version, subject, inputs, *, params=None, atoms=None, context=None): + task = {"schema": "moly-task/1", "stage": {"name": stage, "version": version}, + "subject": subject, "inputs": sorted(inputs, key=lambda item: item["role"]), + "params": params or {}, "atoms": atoms or {}, "context": context or {}} + task["key"] = document_hash(key_parts(task)) + return validate(task) diff --git a/src/pipeline/depot.py b/src/pipeline/depot.py new file mode 100644 index 0000000..f72fabe --- /dev/null +++ b/src/pipeline/depot.py @@ -0,0 +1,86 @@ +"""Run versioned stages and retain their outputs in the existing pack CAS.""" +from pathlib import Path +import tempfile + +from core.atomic import exclusive_lock, write_bytes +from core.content import canonical_bytes, content_hash +from pack.paths import output_path +from pack.task_results import read_result, retained_results +from source.snapshot import read_inputs +from .contract import key_parts, validate + + +class Depot: + def __init__(self, root): + self.root = Path(root).resolve() + + def plan(self, task, *, previous=None): + from .stages import validate_installed + validate(task) + validate_installed(task) + result = read_result(self.root, task["key"]) + reasons = [] + if previous is not None: + validate(previous) + old, new = key_parts(previous), key_parts(task) + reasons = [field + "-changed" for field in new if old[field] != new[field]] + if not result and not reasons: + reasons = ["result-missing"] + return {"schema": "moly-plan/1", "task": task, + "status": "hit" if result else "run", "reasons": reasons or ["cached"]} + + def run(self, task, *, force=False): + from .stages import execute, validate_installed + validate(task) + validate_installed(task) + with exclusive_lock(self.root / ".publish.lock"): + previous = read_result(self.root, task["key"]) + if previous is not None and not force: + return {"status": "hit", "result": previous} + # Freeze and verify all bytes before calling a reader. A caller + # changing files during extraction cannot contaminate this task. + inputs = read_inputs(task["inputs"]) + with tempfile.TemporaryDirectory(prefix="moly-task-") as temp: + artifacts = execute(task, inputs, Path(temp)) + validate_installed(task) + rows = [] + for logical, data in sorted(artifacts.items()): + output_path(self.root, logical) + digest = content_hash(data) + blob = f"{digest[:2]}/{digest}.bin" + path = output_path(self.root, "blobs/" + blob) + if path.exists(): + if path.read_bytes() != data: + raise ValueError("corrupt artifact at content address") + else: + write_bytes(path, data) + rows.append({"path": logical, "sha256": digest, "size": len(data), "blob": blob}) + validate_installed(task) + result = {"schema": "moly-result/1", "key": task["key"], + "keyParts": key_parts(task), "status": "ok", "artifacts": rows} + if previous is not None and result != previous: + raise ValueError("stage changed output for the same task key; bump its version") + write_bytes(output_path(self.root, f"tasks/{task['key']}.json"), canonical_bytes(result)) + return {"status": "ran", "result": result} + + def materialize(self, key, destination): + destination = Path(destination).resolve() + if destination.is_relative_to(self.root) or self.root.is_relative_to(destination): + raise ValueError("layout and depot must be separate directories") + with exclusive_lock(self.root / ".publish.lock"): + result = read_result(self.root, key) + if result is None: + raise ValueError("task has no committed result") + # Discovery is installed last. New hashed documents are complete + # before an existing reader can see the new root index. + for artifact in sorted(result["artifacts"], key=lambda row: ( + row["path"] == "text/index.json" or row["path"].endswith("/latest.json"))): + write_bytes(output_path(destination, artifact["path"]), + output_path(self.root, "blobs/" + artifact["blob"]).read_bytes()) + # Do not hard-link editable layouts to immutable shared bytes. + return {"key": key, "files": len(result["artifacts"])} + + def verify(self): + with exclusive_lock(self.root / ".publish.lock"): + roots, blobs = retained_results(self.root) + return {"results": len(roots), "blobs": len(blobs), "bytes": sum(blobs.values())} diff --git a/src/pipeline/stages.py b/src/pipeline/stages.py new file mode 100644 index 0000000..446e680 --- /dev/null +++ b/src/pipeline/stages.py @@ -0,0 +1,171 @@ +"""Built-in stages call the same domain readers as direct exports. + +Only inputs consumed by the selected stage enter its key. Source-module +fingerprints also separate unreleased edits; versions remain the declared +semantic contract and installed dependency versions identify byte readers. +""" +from importlib.metadata import version +import json +from pathlib import Path + +from core.content import canonical_bytes, content_hash, document_hash +from source.snapshot import file_input +from .contract import describe + +VERSIONS = {"talks.extract": 1, "communication.extract": 1, "text.catalog": 1, "text.official": 1, "text.runtime": 1} +MODULES = { + "communication.extract": ("chara/communication_actions.py", "chara/alone_actions.py", "chara/talks.py", "core/master.py", "core/jsonio.py", "core/unity.py"), + "talks.extract": ("chara/talks.py", "chara/common_talks.py", "perf/fixture_talks.py", + "core/master.py", "core/assets/router.py", "core/unity.py"), + "text.catalog": ("chara/text_catalog.py", "core/jsonio.py"), + "text.official": ("chara/text_catalog.py",), + "text.runtime": ("pipeline/text_product.py",), +} +TALK_TABLES = ( + "mysekaiCharacterTalks", "mysekaiGameCharacterUnitGroups", + "mysekaiCharacterTalkConditions", "mysekaiCharacterTalkConditionGroups", + "mysekaiCharacterTalkPreActions", "mysekaiCharacterTalkTweets", +) + + +def atoms(stage): + root = Path(__file__).resolve().parents[1] + modules = (*MODULES[stage], "pipeline/stages.py", "core/content.py") + result = {"implementation": document_hash({name: content_hash((root / name).read_bytes()) + for name in modules})} + if stage in {"talks.extract", "communication.extract"}: + result["unitypy"] = version("UnityPy") + return result + + +def task(stage, subject, inputs, **params): + return describe(stage, VERSIONS[stage], subject, inputs, params=params, atoms=atoms(stage)) + + +def talk_task(master, bundle, lib, *, family="general", unity_version="2022.3.62f3"): + if family not in {"general", "fixture"}: + raise ValueError("unknown conversation family") + inputs = [file_input("bundle", bundle), file_input("lib", lib)] + tables = list(TALK_TABLES) + if family == "fixture": + tables += ["mysekaiCharacterTalkNoTalkMysekaiFixtureActions", "mysekaiFixtures"] + else: + # Optional tables are part of the input identity when present. Their + # absence is represented by the table list, not a made-up empty file. + tables += [name for name in ("birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks") + if (Path(master) / (name + ".json")).is_file()] + inputs += [file_input("master/" + name, Path(master) / (name + ".json")) for name in tables] + return task("talks.extract", family, inputs, family=family, unityVersion=unity_version) + + +def catalog_task(corpora, *, region, source_revision, previous=None): + inputs = [file_input(f"corpus/{i:04d}", path) for i, path in enumerate(corpora)] + if previous: + inputs.append(file_input("previous", previous)) + return task("text.catalog", region, inputs, region=region, sourceRevision=source_revision) + + +def communication_task(master, bundle, *, unity_version="2022.3.62f3"): + return task("communication.extract", "characters", [file_input("bundle", bundle), + file_input("master/mysekaiCharacterTalkTogetherCommunications", Path(master) / "mysekaiCharacterTalkTogetherCommunications.json")], + unityVersion=unity_version) + + +def official_task(source, donor, *, locale): + return task("text.official", locale, + [file_input("source", source), file_input("donor", donor)], locale=locale) + + +def runtime_task(source, overlays=()): + inputs = [file_input("source", source)] + inputs += [file_input(f"overlay/{i:04d}", path) for i, path in enumerate(overlays)] + return task("text.runtime", "display", inputs) + + +def validate_installed(value): + name = value["stage"]["name"] + if name not in VERSIONS or value["stage"]["version"] != VERSIONS[name]: + raise ValueError("task stage/version is not installed") + if value["atoms"] != atoms(name): + raise ValueError("task implementation differs from this installation; regenerate the plan") + expected = {"talks.extract": {"family", "unityVersion"}, "communication.extract": {"unityVersion"}, + "text.catalog": {"region", "sourceRevision"}, + "text.official": {"locale"}, "text.runtime": set()} + if set(value["params"]) != expected[name]: + raise ValueError("task parameters differ from the stage contract") + roles = {item["role"] for item in value["inputs"]} + if name == "communication.extract": + if roles != {"bundle", "master/mysekaiCharacterTalkTogetherCommunications"}: + raise ValueError("communication requires bundle and source master") + elif name == "talks.extract": + if value["params"]["family"] not in {"general", "fixture"}: + raise ValueError("unknown conversation family") + permitted = set(TALK_TABLES) | {"birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", + "mysekaiFixtures", "mysekaiCharacterTalkNoTalkMysekaiFixtureActions"} + if not {"bundle", "lib"} <= roles or any( + role not in {"bundle", "lib"} and role not in {"master/" + t for t in permitted} + for role in roles): + raise ValueError("unknown or missing talk input roles") + elif name == "text.official" and roles != {"source", "donor"}: + raise ValueError("official text requires source and donor") + elif name in {"text.catalog", "text.runtime"}: + prefix = "corpus/" if name == "text.catalog" else "overlay/" + singles = {"previous"} if name == "text.catalog" else {"source"} + if any(role not in singles and not (role.startswith(prefix) and role[len(prefix):].isdigit()) + for role in roles): + raise ValueError("unknown stage input role") + if (name == "text.runtime" and "source" not in roles + or name == "text.catalog" and not any(role.startswith(prefix) for role in roles)): + raise ValueError("stage has no source inputs") + + +def execute(value, inputs, workspace): + name, params = value["stage"]["name"], value["params"] + if name == "communication.extract": + from chara.communication_actions import extract_communication_actions, TABLE + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(params["unityVersion"]) + master = workspace / "master" + master.mkdir() + (master / (TABLE + ".json")).write_bytes(inputs["master/" + TABLE]) + bundle, out = workspace / "bundle", workspace / "communication-actions.json" + bundle.write_bytes(inputs["bundle"]) + extract_communication_actions(master, bundle, out) + return {"communication-actions.json": out.read_bytes()} + if name == "talks.extract": + from core.unity import configure_fallback_unity_version + from chara.talks import extract_talks + from perf.fixture_talks import extract_fixture_talks + configure_fallback_unity_version(params["unityVersion"]) + master = workspace / "master" + master.mkdir() + for role, data in inputs.items(): + if role.startswith("master/"): + (master / (role.removeprefix("master/") + ".json")).write_bytes(data) + bundle, lib, out = workspace / "bundle", workspace / "lib", workspace / "talks.json" + bundle.write_bytes(inputs["bundle"]) + lib.write_bytes(inputs["lib"]) + reader = extract_talks if params["family"] == "general" else extract_fixture_talks + reader(str(master), str(bundle), str(out), lib_bundle=str(lib)) + doc = json.loads(out.read_bytes()) + if params["family"] == "fixture": + summary = doc["summary"] + if summary.get("missingScripts") or summary.get("unknownOperations"): + raise ValueError("fixture conversation extraction is incomplete") + return {("talks.json" if params["family"] == "general" else "fixture-talks/talks.json"): + out.read_bytes()} + if name == "text.catalog": + from chara.text_catalog import build_catalog + doc = build_catalog([json.loads(data) for role, data in sorted(inputs.items()) if role.startswith("corpus/")], + region=params["region"], source_revision=params["sourceRevision"], + previous=json.loads(inputs["previous"]) if "previous" in inputs else None) + return {"text-source.json": canonical_bytes(doc)} + if name == "text.official": + from chara.text_catalog import match_official + doc = match_official(json.loads(inputs["source"]), json.loads(inputs["donor"]), locale=params["locale"]) + return {f"official-{params['locale']}.json": canonical_bytes(doc)} + if name == "text.runtime": + from .text_product import build_product + return build_product(json.loads(inputs["source"]), + [json.loads(data) for role, data in sorted(inputs.items()) if role.startswith("overlay/")]) + raise ValueError("unknown stage") diff --git a/src/pipeline/text_product.py b/src/pipeline/text_product.py new file mode 100644 index 0000000..fd7f1f6 --- /dev/null +++ b/src/pipeline/text_product.py @@ -0,0 +1,95 @@ +"""Small per-content display documents, with a bounded two-level index. + +The complete source registry stays in editorial storage. A player reads the +root, one routing bucket and the selected script's display/overlay documents. +Each script includes its linked tweets so a session fixes both together. +""" +from collections import defaultdict + +from core.content import canonical_bytes, content_hash, document_hash + +MAX_SHARD_BYTES = 2 * 1024 * 1024 + + +def build_product(source, overlays=()): + if source.get("schema") != "moly-text-source/1": + raise ValueError("unsupported text source schema") + scripts = {row["key"]: row for row in source["scripts"]} + if len(scripts) != len(source["scripts"]): + raise ValueError("duplicate text script identity") + all_lines = {line["id"]: line for row in scripts.values() for line in row["lines"]} + if len(all_lines) != sum(len(row["lines"]) for row in scripts.values()): + raise ValueError("duplicate text line identity") + locales = defaultdict(dict) + for overlay in overlays: + if (overlay.get("schema") != "moly-text-overlay/1" + or overlay.get("sourceRegion") != source["region"] + or overlay.get("sourceRevision") != source["revision"]): + raise ValueError("overlay does not belong to this source revision") + for entry in overlay["entries"]: + line = all_lines.get(entry["id"]) + if line is None or entry["sourceHash"] != line["sourceHash"]: + raise ValueError("overlay refers to a missing or changed source line") + if entry["id"] in locales[overlay["locale"]]: + raise ValueError("duplicate overlay line for the same locale") + locales[overlay["locale"]][entry["id"]] = entry + files, buckets = {}, defaultdict(dict) + translations = {locale: defaultdict(dict) for locale in locales} + + def store(document, prefix="text"): + data = canonical_bytes(document) + if len(data) > MAX_SHARD_BYTES: + raise ValueError("text shard exceeds the runtime document limit") + digest = content_hash(data) + path = f"{prefix}/objects/{digest}.json" + files[path] = data + return {"path": f"objects/{digest}.json", "sha256": digest, "bytes": len(data)} + + for key, script in sorted(scripts.items()): + tweet_keys = {c["tweetKey"] for c in script["contents"] if c.get("tweetKey")} + if tweet_keys - scripts.keys(): + raise ValueError("missing linked tweet source") + selected = [script] + [scripts[k] for k in sorted(tweet_keys) if k != key] + display = {"schema": "moly-text-source/1", "region": source["region"], + "scripts": [{"key": row["key"], "lines": [ + {field: line[field] for field in ("id", "kind", "step", "text", "sourceHash")} + for line in row["lines"]]} for row in selected]} + display["revision"] = document_hash(display) + route = {"source": store(display)} + bucket = content_hash(key.encode("utf-8"))[:2] + ids = {line["id"] for row in selected for line in row["lines"]} + for locale, entries in sorted(locales.items()): + # Donor/global revision provenance stays in the source registry + # and root. A change to another conversation must not churn this + # immutable per-script payload. + selected_entries = [{field: entries[identity][field] + for field in ("id", "sourceHash", "text", "source")} + for identity in sorted(ids & entries.keys())] + if not selected_entries: + continue + doc = {"schema": "moly-text-overlay/1", "sourceRegion": source["region"], + "sourceRevision": display["revision"], "locale": locale, "entries": selected_entries} + doc["revision"] = document_hash(doc) + prefix = f"translations/{source['region']}/{locale}" + ref = store(doc, prefix) + translations[locale][bucket][key] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + buckets[bucket][key] = route + root = {"schema": "moly-text-runtime/1", "region": source["region"], + "sourceRevision": source["revision"], "routing": "sha256-utf8-prefix-2", + "buckets": {}} + for bucket, routes in sorted(buckets.items()): + root["buckets"][bucket] = store({"schema": "moly-text-routes/1", "scripts": routes}) + root["revision"] = document_hash(root) + files["text/index.json"] = canonical_bytes(root) + for locale, translation_buckets in sorted(translations.items()): + prefix = f"translations/{source['region']}/{locale}" + publication = {"schema": "moly-text-publication/1", "region": source["region"], + "locale": locale, "routing": "sha256-utf8-prefix-2", "buckets": {}} + for bucket, routes in sorted(translation_buckets.items()): + ref = store({"schema": "moly-text-translation-routes/1", "scripts": routes}, prefix) + publication["buckets"][bucket] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + publication["revision"] = document_hash(publication) + data = canonical_bytes(publication) + files[f"{prefix}/revisions/{publication['revision']}.json"] = data + files[f"{prefix}/latest.json"] = data + return files diff --git a/src/source/__init__.py b/src/source/__init__.py new file mode 100644 index 0000000..b75d8ec --- /dev/null +++ b/src/source/__init__.py @@ -0,0 +1 @@ +"""Caller-supplied, content-verified sources used by extraction tasks.""" diff --git a/src/source/snapshot.py b/src/source/snapshot.py new file mode 100644 index 0000000..d94e60c --- /dev/null +++ b/src/source/snapshot.py @@ -0,0 +1,47 @@ +"""Portable source receipts: content identity is independent of its locator. + +Receipts and plans can contain local paths. They are private build inputs, +not runtime artifacts. A task verifies the bytes it actually reads instead +of trusting file times or a previously computed hash. +""" +from pathlib import Path + +from core.content import content_hash, document_hash + + +def file_input(role, path): + path = Path(path).resolve(strict=True) + data = path.read_bytes() + return {"role": role, "sha256": content_hash(data), "size": len(data), + "locators": [{"kind": "file", "path": str(path)}]} + + +def read_inputs(inputs): + frozen = {} + for item in inputs: + failures = [] + for locator in item["locators"]: + try: + if locator.get("kind") != "file": + raise ValueError("unsupported input locator") + data = Path(locator["path"]).read_bytes() + if len(data) != item["size"] or content_hash(data) != item["sha256"]: + raise ValueError("input content changed after planning") + frozen[item["role"]] = data + break + except (OSError, ValueError) as exc: + failures.append(str(exc)) + else: + raise ValueError(f"cannot read verified input {item['role']}: {'; '.join(failures)}") + return frozen + + +def snapshot(inputs, *, region, version): + if region not in {"jp", "cn", "tw", "en", "kr"} or not version: + raise ValueError("source requires a known region and an explicit version") + inputs = sorted(inputs, key=lambda item: item["role"]) + if len({item["role"] for item in inputs}) != len(inputs): + raise ValueError("duplicate source input role") + identity = {"schema": "moly-source/1", "region": region, "version": version, + "inputs": [{k: item[k] for k in ("role", "sha256", "size")} for item in inputs]} + return {**identity, "revision": document_hash(identity), "inputs": inputs} diff --git a/tests/check_distribution.py b/tests/check_distribution.py index 31711ec..3d3c2e2 100644 --- a/tests/check_distribution.py +++ b/tests/check_distribution.py @@ -56,7 +56,7 @@ def run(name, command): moly = scripts / ("moly.exe" if os.name == "nt" else "moly") run("install", [str(python), "-m", "pip", "install", str(wheel)]) run("dependencies", [str(python), "-m", "pip", "check"]) - for command in ([], ["pull"], ["extract"], ["fetch-apk"]): + for command in ([], ["pull"], ["extract"], ["fetch-apk"], ["source"], ["depot"], ["export", "text-runtime"]): run("help-" + (command[0] if command else "moly"), [str(moly), *command, "--help"]) runtime = r''' diff --git a/tests/test_birthday_party_talks.py b/tests/test_birthday_party_talks.py index f28f8a9..101d332 100644 --- a/tests/test_birthday_party_talks.py +++ b/tests/test_birthday_party_talks.py @@ -117,10 +117,9 @@ def test_an_absent_party_table_is_stated_not_silently_empty(tmp_path, monkeypatc assert "birthdayPartyScenarios" in block["reason"] -def test_the_party_block_is_the_only_key_the_change_adds(tmp_path, monkeypatch): - # The ordinary corpus is the document it always was, plus one key: a reader - # that held the top-level keys as a closed set would see exactly one new - # name, and no key it already had may be renamed, dropped or reordered. +def test_party_and_common_blocks_preserve_the_ordinary_corpus_keys(tmp_path, monkeypatch): + # Additional families have their own blocks; ordinary corpus keys stay + # present and ordered before them. monkeypatch.setattr(talks, "_text_assets", lambda bundle: _assets()) master = _master(tmp_path, [{"id": 1, "birthdayPartyId": 1, "seq": 1, "lua": "party_001"}]) @@ -128,7 +127,7 @@ def test_the_party_block_is_the_only_key_the_change_adds(tmp_path, monkeypatch): talks.extract_talks(master, "synthetic.bundle", out) document = json.loads(out.read_text(encoding="utf-8")) assert list(document) == ["version", "semantics", "units", "summary", - "birthdayParty"] + "birthdayParty", "commonSomeCharacter"] assert document["birthdayParty"]["semantics"]["table"].startswith( "birthdayPartyScenarios") @@ -175,7 +174,8 @@ def test_attach_leaves_every_other_key_alone(tmp_path, monkeypatch): after = json.loads(out.read_text(encoding="utf-8")) assert after["birthdayParty"]["status"] == "present" assert after["birthdayParty"]["talks"]["party_001"]["voices"] == ["voice_party_001_01_006"] - assert report["preservedKeys"] == ["version", "semantics", "units", "summary"] + assert report["preservedKeys"] == ["version", "semantics", "units", "summary", + "commonSomeCharacter"] # Byte for byte: with the block taken back out, the attached document is # the corpus the extraction wrote, minus that run's own absent block. del after["birthdayParty"] diff --git a/tests/test_common_talks.py b/tests/test_common_talks.py new file mode 100644 index 0000000..03609ee --- /dev/null +++ b/tests/test_common_talks.py @@ -0,0 +1,75 @@ +"""Source-predicate probes over caller-supplied synthetic master tables. + +The source accepts two through five members, tests only the fixture-id +condition, and requires a matching association. Keep the boundary cases +alongside positive rows so dropping the whole family cannot satisfy them. +""" +import json + +from chara.common_talks import common_talks, select_common_talks +from core.master import Master +from phenomena.audio_corpus import voice_requests + + +def source(tmp_path): + def write(name, rows): + (tmp_path / f"{name}.json").write_text(json.dumps(rows), encoding="utf-8") + + groups = [] + for size in range(1, 6): + groups.append({"id": size, **{f"gameCharacterUnitId{i}": i for i in range(1, size + 1)}}) + write("mysekaiGameCharacterUnitGroups", groups) + kinds = ["mysekai_character_visit_count", "mysekai_fixture_id", "mysekai_fixture_tag_id"] + write("mysekaiCharacterTalkConditions", [ + {"id": i, "mysekaiCharacterTalkConditionType": kind, + "mysekaiCharacterTalkConditionTypeValue": 1} + for i, kind in enumerate(kinds, 1)]) + write("mysekaiCharacterTalkConditionGroups", [ + {"id": i, "groupId": i, "mysekaiCharacterTalkConditionId": i} for i in range(1, 4)]) + rows = [ + {"id": identity, "mysekaiGameCharacterUnitGroupId": size, + "mysekaiCharacterTalkConditionGroupId": condition, + "mysekaiSiteGroupId": 1, "mysekaiCharacterTalkTermId": 1, "lua": f"sample_{identity}"} + for identity, size, condition in [(1, 1, 1), (2, 2, 1), (3, 5, 1), + (4, 2, 2), (5, 2, 3), (6, 2, 1)]] + write("mysekaiCharacterTalks", rows) + write("mysekaiCharacterTalkSomeCharacterTalks", [ + {"id": identity, "mysekaiCharacterTalkId": identity, "mainGameCharacterUnitId": 2} + for identity in range(1, 6)]) + write("mysekaiCharacterTalkPreActions", []) + write("mysekaiCharacterTalkTweets", []) + return Master(tmp_path) + + +def test_native_membership_boundaries_and_exact_fixture_predicate(tmp_path): + rows, report = select_common_talks(source(tmp_path)) + assert [row["talk"]["id"] for row in rows] == [2, 3, 5] + assert rows[1]["unitIds"] == [1, 2, 3, 4, 5] + assert report["withoutAssociation"] == [6] + + +def test_shared_parser_preserves_both_speakers_and_routes_common_voice_banks(tmp_path): + scripts = {f"sample_{i}.lua": f'''label("Alpha") +voice("talk", "voice_mysekai_sample_{i}_01_001", Characters.Alpha) +text("First") +wait_click() +label("Beta") +voice("talk", "voice_mysekai_sample_{i}_02_002", Characters.Beta) +text("Second") +wait_click() +''' for i in [2, 3, 5]} + block, report = common_talks(source(tmp_path), scripts, {"Characters": {"Alpha": 1, "Beta": 2}}, {}) + assert report["talks"] == 3 + assert [s["who"] for s in block["talks"][0]["steps"] if s["op"] == "voice"] == [1, 2] + assert block["talks"][0]["someCharacterTalks"][0]["mainGameCharacterUnitId"] == 2 + assert "mainUnitId" not in block["talks"][0] + requests, cues = voice_requests({"commonSomeCharacter": block}) + assert len(requests) == 3 + assert len(cues) == 6 + + +def test_unrecognized_source_call_is_reported_before_export(tmp_path): + import pytest + scripts = {"sample_2.lua": 'new_source_operation("value")'} + with pytest.raises(ValueError, match="unknown script operations"): + common_talks(source(tmp_path), scripts, {}, {}) diff --git a/tests/test_motion_storage.py b/tests/test_motion_storage.py index cac9dd2..676ea55 100644 --- a/tests/test_motion_storage.py +++ b/tests/test_motion_storage.py @@ -205,7 +205,8 @@ def export(reference, bundle, output, name, **options): @pytest.mark.parametrize("override", [[], ["--rotation-encoding", "float32"]]) def test_export_cli_forwards_rotation_encoding(monkeypatch, override): - from core import cli, extract, runtime_pack + from cli import legacy as cli + from core import extract, runtime_pack expected = "float32" if override else "snorm16" monkeypatch.setattr(cli, "_configure_unity", lambda _: None) calls = [] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..688f649 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,153 @@ +"""Portable task execution, corruption boundaries and runtime text closure.""" +from copy import deepcopy +import json +from pathlib import Path + +import pytest + +from core.content import canonical_bytes, content_hash +from pack.gc import catalog_garbage +from pipeline.depot import Depot +from pipeline.stages import catalog_task, official_task, runtime_task + + +def corpus(text="Hello"): + return {"commonSomeCharacter": {"talks": [{"lua": "sample", "talkId": 1, "unitIds": [1, 2], + "steps": [{"op": "label", "name": "Alpha"}, {"op": "voice", "who": 1, "cue": "one"}, + {"op": "text", "text": text}, {"op": "wait_click"}]}]}} + + +def setup(tmp_path): + source = tmp_path / "input.json" + source.write_bytes(canonical_bytes(corpus())) + task = catalog_task([source], region="jp", source_revision="sample-v1") + return source, task, Depot(tmp_path / "store") + + +def test_relocated_inputs_and_outputs_share_a_verified_result(tmp_path): + source, task, depot = setup(tmp_path) + assert depot.plan(task)["status"] == "run" + first = depot.run(task) + relocated = tmp_path / "elsewhere.json" + relocated.write_bytes(source.read_bytes()) + same = catalog_task([relocated], region="jp", source_revision="sample-v1") + assert same["key"] == task["key"] + assert depot.run(same)["status"] == "hit" + fresh = Depot(tmp_path / "fresh") + assert fresh.run(same)["result"] == first["result"] + assert depot.run(same, force=True)["result"] == first["result"] + depot.materialize(task["key"], tmp_path / "a") + fresh.materialize(task["key"], tmp_path / "b") + assert (tmp_path / "a/text-source.json").read_bytes() == (tmp_path / "b/text-source.json").read_bytes() + assert str(tmp_path) not in canonical_bytes(first["result"]).decode() + + +def test_changed_inputs_invalidate_plan_and_stale_description_cannot_commit(tmp_path): + source, task, depot = setup(tmp_path) + source.write_bytes(canonical_bytes(corpus("Updated"))) + changed = catalog_task([source], region="jp", source_revision="sample-v1") + assert task["key"] != changed["key"] + assert "inputs-changed" in depot.plan(changed, previous=task)["reasons"] + with pytest.raises(ValueError, match="content changed"): + depot.run(task) + assert not (depot.root / "tasks" / f"{task['key']}.json").exists() + assert depot.run(changed)["status"] == "ran" + + +def test_corrupt_cache_is_neither_hit_nor_collectable(tmp_path): + _, task, depot = setup(tmp_path) + result = depot.run(task)["result"] + blob = depot.root / "blobs" / result["artifacts"][0]["blob"] + assert catalog_garbage(depot.root)["delete"] == [] + blob.write_bytes(b"broken") + with pytest.raises(ValueError, match="corrupt"): + depot.plan(task) + with pytest.raises(ValueError, match="corrupt"): + catalog_garbage(depot.root) + + +def test_task_hash_and_installed_reader_are_checked(tmp_path): + _, task, depot = setup(tmp_path) + changed = deepcopy(task) + changed["params"]["region"] = "cn" + with pytest.raises(ValueError, match="key"): + depot.run(changed) + from pipeline.contract import key_parts + from core.content import document_hash + changed = deepcopy(task) + changed["atoms"]["implementation"] = "0" * 64 + changed["key"] = document_hash(key_parts(changed)) + with pytest.raises(ValueError, match="installation"): + depot.run(changed) + + +def test_published_release_gc_retains_independent_task_artifacts(tmp_path): + from pack.groups import build_groups, runtime_paths + _, task, depot = setup(tmp_path) + result = depot.run(task)["result"] + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "manifest.json").write_text('{"units": []}', encoding="utf-8") + (runtime / "characters.json").write_text('{}', encoding="utf-8") + (runtime / "text").mkdir() + (runtime / "text/index.json").write_text('{}', encoding="utf-8") + (runtime / "translations").mkdir() + (runtime / "translations/latest.json").write_text('{}', encoding="utf-8") + selected = {str(path).replace("\\", "/") for path in runtime_paths(runtime)} + assert any(path.endswith("text/index.json") for path in selected) + assert not any("translations" in path for path in selected) + build_groups(runtime, depot.root, "synthetic", region="jp") + garbage = catalog_garbage(depot.root) + assert garbage["delete"] == [] + assert f"tasks/{task['key']}.json" in garbage["roots"] + assert result["artifacts"][0]["blob"] in garbage["referenced_blobs"] + + +def test_runtime_only_loads_content_closure_and_matching_official_text(tmp_path): + source, jp_task, depot = setup(tmp_path) + depot.run(jp_task) + depot.materialize(jp_task["key"], tmp_path / "jp") + source.write_bytes(canonical_bytes(corpus("你好"))) + cn_task = catalog_task([source], region="cn", source_revision="cn-v1") + depot.run(cn_task) + depot.materialize(cn_task["key"], tmp_path / "cn") + jp, cn = tmp_path / "jp/text-source.json", tmp_path / "cn/text-source.json" + official = official_task(jp, cn, locale="zh-CN") + depot.run(official) + depot.materialize(official["key"], tmp_path / "official") + runtime = runtime_task(jp, [tmp_path / "official/official-zh-CN.json"]) + depot.run(runtime) + out = tmp_path / "runtime" + depot.materialize(runtime["key"], out) + + def read(ref): + data = (out / ref["path"]).read_bytes() + assert content_hash(data) == ref["sha256"] and len(data) == ref["bytes"] + return json.loads(data) + + index = json.loads((out / "text/index.json").read_bytes()) + bucket = content_hash(b"sample")[:2] + route = read({**index["buckets"][bucket], "path": "text/" + index["buckets"][bucket]["path"]})["scripts"]["sample"] + script = read({**route["source"], "path": "text/" + route["source"]["path"]}) + base = "translations/jp/zh-CN/" + publication = json.loads((out / base / "latest.json").read_bytes()) + translation_route = read({**publication["buckets"][bucket], + "path": base + publication["buckets"][bucket]["path"]})["scripts"]["sample"] + translated = read({**translation_route, "path": base + translation_route["path"]}) + assert [entry["text"] for entry in translated["entries"]] == ["你好"] + assert translated["sourceRevision"] == script["revision"] + assert all(set(line) == {"id", "kind", "step", "text", "sourceHash"} + for row in script["scripts"] for line in row["lines"]) + assert all("steps" not in row for row in script["scripts"]) + + # Translation publication changes have no path back into a resource + # snapshot, its source index, or any unchanged source shard. + from pipeline.text_product import build_product + source_doc = json.loads(jp.read_bytes()) + overlay_doc = json.loads((tmp_path / "official/official-zh-CN.json").read_bytes()) + before = build_product(source_doc, [overlay_doc]) + overlay_doc["entries"][0]["text"] = "您好" + after = build_product(source_doc, [overlay_doc]) + assert {p: b for p, b in before.items() if p.startswith("text/")} == { + p: b for p, b in after.items() if p.startswith("text/")} + assert before[base + "latest.json"] != after[base + "latest.json"] diff --git a/tests/test_text_catalog.py b/tests/test_text_catalog.py new file mode 100644 index 0000000..05ebb7f --- /dev/null +++ b/tests/test_text_catalog.py @@ -0,0 +1,84 @@ +"""Identity and official-text alignment tests with authored synthetic scripts.""" +from copy import deepcopy + +from chara.text_catalog import build_catalog, match_official + + +def document(texts=("Hello", "Again"), *, members=(1, 2)): + steps = [] + for index, text in enumerate(texts): + steps.extend([{"op": "label", "name": "Alpha"}, + {"op": "voice", "who": 1, "cue": f"cue_{index}"}, + {"op": "text", "text": text}, {"op": "wait_click"}]) + return {"commonSomeCharacter": {"talks": [{ + "talkId": 1, "lua": "sample", "unitIds": list(members), "steps": steps, + "tweet": {"id": 1, "text": "Over here!"}, + }]}} + + +def catalog(doc, *, region="jp", previous=None): + return build_catalog([doc], region=region, source_revision="sample-1", previous=previous) + + +def lines(value): + return value["scripts"][0]["lines"] + + +def test_motion_insertion_changes_binding_not_text_identity(): + doc = document() + before = catalog(doc) + changed = deepcopy(doc) + changed["commonSomeCharacter"]["talks"][0]["steps"].insert(0, {"op": "wait_time", "seconds": 2}) + after = catalog(changed, previous=before) + assert [r["id"] for r in lines(after)] == [r["id"] for r in lines(before)] + assert lines(after)[0]["step"] == lines(before)[0]["step"] + 1 + + +def test_source_edit_keeps_voice_anchored_id_but_invalidates_translation_hash(): + before = catalog(document()) + after = catalog(document(("Changed", "Again")), previous=before) + first_before, first_after = lines(before)[1], lines(after)[1] + assert first_before["id"] == first_after["id"] + assert first_before["sourceHash"] != first_after["sourceHash"] + assert first_after["change"] == "changed" + + +def test_ambiguous_unvoiced_repetitions_after_edit_get_new_ids(): + doc = document() + row = doc["commonSomeCharacter"]["talks"][0] + row["steps"] = [{"op": "text", "text": "Same"}, {"op": "text", "text": "Same"}] + before = catalog(doc) + row["steps"].insert(1, {"op": "text", "text": "New"}) + after = catalog(doc, previous=before) + assert not {r["id"] for r in lines(before)} & {r["id"] for r in lines(after)} + assert len(after["scripts"][0]["retiredLines"]) == 2 + + +def test_reimporting_same_catalogue_preserves_revision_and_retired_history(): + before = catalog(document()) + after = catalog(document(), previous=before) + assert after["revision"] == before["revision"] + + +def test_same_script_can_have_distinct_tweet_bindings(): + doc = document() + row = deepcopy(doc["commonSomeCharacter"]["talks"][0]) + row["talkId"] = 2 + row["tweet"] = {"id": 2, "text": "Hello there!"} + doc["commonSomeCharacter"]["talks"].append(row) + result = catalog(doc) + assert len(result["scripts"]) == 3 + assert len(result["scripts"][0]["contents"]) == 2 + assert result["scripts"][0]["contents"][1]["tweetKey"] == "@tweet/2" + + +def test_official_text_requires_matching_cast_and_per_line_voice_sequence(): + source = catalog(document()) + donor = catalog(document(("你好", "再见")), region="cn") + paired = match_official(source, donor, locale="zh-CN") + assert [r["text"] for r in paired["entries"]] == ["你好", "再见"] + wrong_cast = catalog(document(("你好", "再见"), members=(1, 3)), region="cn") + assert not match_official(source, wrong_cast, locale="zh-CN")["entries"] + wrong_voice = document(("你好", "再见")) + wrong_voice["commonSomeCharacter"]["talks"][0]["steps"][1]["cue"] = "different" + assert not match_official(source, catalog(wrong_voice, region="cn"), locale="zh-CN")["entries"] diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 7bba12c..6787689 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -220,7 +220,7 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) is_snapshot = len(parts) >= 3 and parts[0] == 'snapshots' and parts[1] in snapshots # The shared content-addressed stores are immutable like snapshot payloads. shared_store = len(parts) >= 2 and parts[0] in {'asset-store', 'catalog-store'} - choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog'} and len(parts) >= 4) + choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog', 'text'} and len(parts) >= 4) or shared_store)) or (args.scope == 'descriptors' and is_snapshot and len(parts) == 3 and parts[2] in {'snapshot.json', 'provenance.json'}) or (args.scope == 'release' and len(parts) >= 3 and parts[:2] == ('releases', args.release_id)) diff --git a/tools/migrate_canonical_metadata.py b/tools/migrate_canonical_metadata.py index c7831e4..cd15c22 100644 --- a/tools/migrate_canonical_metadata.py +++ b/tools/migrate_canonical_metadata.py @@ -31,7 +31,10 @@ 'src/chara/avatar_parts.py': '74a42b5519bae8b645850e0281420c707ebac33fa08df3f5bd6326a4dcf89fc3', 'src/chara/avatar_compose.py': '366c5f5e1253728dbbcbb6f476f26dfddaa2062cbe7a6e0ccab5bffd85ec4ca7', 'src/chara/sd_switch.py': '3943cde0837981f94dc9721271313a96dae6a6df6bc928802e2bd82a1f0d473c', - 'src/perf/animations.py': '00f0f58f26895fb5b2163a788ba95a424fd8e0b36052ab2928b77fb1948afe47', + # Only the CLI's default output location changed (MOLY_EXTRACT_OUT); + # reverting that return expression reproduces the previous reviewed hash. + # The coordinate-writing implementation is byte-identical. + 'src/perf/animations.py': '16e31e5b719d47bc04077db95236f3e1efa4295d96fe57433a03f94154e9e851', 'src/core/mesh.py': 'e310116a5d0a905f9cb4f89e2e6c8a5057fd1feb86ded789f00abc4d076fa882', 'src/phenomena/meshes.py': 'd4a20e8f9b0b1d0b9ce7a00ea52ebd63745851c5b9d4b28a6b2d39a46f481a8f', 'src/core/gltf.py': '11129743aab66fa0ed7c6997c7f779f1a06d4cc88a021bfb17a1a42d506451fd', From 55f1dfe3391c3a55c9dd9383f63e7a4f3d3ea8b7 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:47:56 +0800 Subject: [PATCH 02/15] feat(text): publish independent search and translation delivery --- docs/pipeline.md | 28 ++++++ src/pipeline/text_product.py | 18 +++- src/ui/jp_decoders.py | 5 ++ src/ui/photo_layout.py | 53 +++++++++++ tests/test_jp_decoders.py | 4 +- tests/test_s3_deployment.py | 33 +++++++ tools/deployment/prepare_translations.py | 107 +++++++++++++++++++++++ tools/deployment/upload_oss.py | 40 ++++++++- tools/deployment/upload_s3.py | 2 +- 9 files changed, 280 insertions(+), 10 deletions(-) create mode 100644 src/ui/photo_layout.py create mode 100644 tools/deployment/prepare_translations.py diff --git a/docs/pipeline.md b/docs/pipeline.md index 02fc97e..184810b 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -97,6 +97,34 @@ and source ID/hash compatibility is checked per line, not by requiring the whole source registry revision to match. Retain historical publication roots and their objects for playback pins and rollback. +Catalogue browsing uses optional `searchBuckets`: 16 shards selected by the +first hash digit. Source rows are `[id, sourceHash, kind, text]`; translation +rows are `[id, sourceHash, text, source]`, grouped by script. These compact +indexes let hosts translate titles and search all dialogue without fetching +each playback document. Source and translation indexes remain in their +respective publications. A translation edit changes its leaf, route and search +shard; unrelated shards and all resource snapshot bytes remain identical. + +Prepare official text for the existing S3 uploader independently: + +```sh +python tools/deployment/prepare_translations.py \ + --publication display/translations/jp/zh-CN --output translation-staging \ + --object-prefix sekai-extra-assets/ --url-prefix /moly/ +``` + +Use the resulting `upload-manifest.json` with `upload_s3.py` and the existing +destination options. `--scope stable` selects immutable objects and the archived +revision root. After these have been uploaded and verified, `--scope mutable +--mutable-path translations/jp/zh-CN/latest.json --allow-mutable-update` selects +only the current pointer. Replacing an existing pointer also requires +`--expect-previous-sha256` for the exact previous bytes. Dry-run remains the +default; `--execute --exclusive-writer` explicitly enables writes. Retain old +objects and roots for rollback; ordinary resource deployment excludes this tree. +This preparation path handles text-only publications; additional font delivery +requires a separately reviewed font publication rather than silently omitting +font references. + These exports prepare inputs for the editorial and playback systems. They do not publish translations or grant approval to drafts. diff --git a/src/pipeline/text_product.py b/src/pipeline/text_product.py index fd7f1f6..c3d0606 100644 --- a/src/pipeline/text_product.py +++ b/src/pipeline/text_product.py @@ -33,8 +33,9 @@ def build_product(source, overlays=()): if entry["id"] in locales[overlay["locale"]]: raise ValueError("duplicate overlay line for the same locale") locales[overlay["locale"]][entry["id"]] = entry - files, buckets = {}, defaultdict(dict) + files, buckets, search = {}, defaultdict(dict), defaultdict(dict) translations = {locale: defaultdict(dict) for locale in locales} + translated_search = {locale: defaultdict(dict) for locale in locales} def store(document, prefix="text"): data = canonical_bytes(document) @@ -57,6 +58,8 @@ def store(document, prefix="text"): display["revision"] = document_hash(display) route = {"source": store(display)} bucket = content_hash(key.encode("utf-8"))[:2] + search[bucket[0]][key] = [[line["id"], line["sourceHash"], line["kind"], line["text"]] + for line in script["lines"]] ids = {line["id"] for row in selected for line in row["lines"]} for locale, entries in sorted(locales.items()): # Donor/global revision provenance stays in the source registry @@ -73,21 +76,30 @@ def store(document, prefix="text"): prefix = f"translations/{source['region']}/{locale}" ref = store(doc, prefix) translations[locale][bucket][key] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + own_ids = {line["id"] for line in script["lines"]} + own_entries = [[line["id"], line["sourceHash"], line["text"], line["source"]] + for line in selected_entries if line["id"] in own_ids] + if own_entries: + translated_search[locale][bucket[0]][key] = own_entries buckets[bucket][key] = route root = {"schema": "moly-text-runtime/1", "region": source["region"], "sourceRevision": source["revision"], "routing": "sha256-utf8-prefix-2", - "buckets": {}} + "buckets": {}, "searchBuckets": {}} for bucket, routes in sorted(buckets.items()): root["buckets"][bucket] = store({"schema": "moly-text-routes/1", "scripts": routes}) + for bucket, rows in sorted(search.items()): + root["searchBuckets"][bucket] = store({"schema": "moly-text-source-search/1", "scripts": rows}) root["revision"] = document_hash(root) files["text/index.json"] = canonical_bytes(root) for locale, translation_buckets in sorted(translations.items()): prefix = f"translations/{source['region']}/{locale}" publication = {"schema": "moly-text-publication/1", "region": source["region"], - "locale": locale, "routing": "sha256-utf8-prefix-2", "buckets": {}} + "locale": locale, "routing": "sha256-utf8-prefix-2", "buckets": {}, "searchBuckets": {}} for bucket, routes in sorted(translation_buckets.items()): ref = store({"schema": "moly-text-translation-routes/1", "scripts": routes}, prefix) publication["buckets"][bucket] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + for bucket, rows in sorted(translated_search[locale].items()): + publication["searchBuckets"][bucket] = store({"schema": "moly-text-translation-search/1", "scripts": rows}, prefix) publication["revision"] = document_hash(publication) data = canonical_bytes(publication) files[f"{prefix}/revisions/{publication['revision']}.json"] = data diff --git a/src/ui/jp_decoders.py b/src/ui/jp_decoders.py index b36216f..780b6f6 100644 --- a/src/ui/jp_decoders.py +++ b/src/ui/jp_decoders.py @@ -141,6 +141,7 @@ from ui import bgm_select_layout from ui import mission_layout from ui import convert_layout +from ui import photo_layout REGION = "jp" @@ -534,6 +535,7 @@ def decode_list_view_700(r: talk.Reader) -> dict: # The 7.0.0 layout: the four dialogs end at their click event, and the five # classes the 7.0.0 declaration changed read their added fields. ONLY_700 = { + **photo_layout.DECODERS, "Sekai.UI.CustomScrollRect": decode_custom_scroll_rect_700, "Sekai.Mysekai.MysekaiMissionStartDialog": mission_layout.decode_mission_start_dialog, "Sekai.Mysekai.MysekaiMissionCompleteDialog": mission_layout.decode_mission_complete_dialog, @@ -560,6 +562,9 @@ def decode_list_view_700(r: talk.Reader) -> dict: # (class name, serialized type hash hex) -> the layout the bytes carry. TYPE_LAYOUTS = { + ("Sekai.Mysekai.MysekaiPhotoShotView", "9c249c4e2d9276be8c161151ab811ebf"): "7.0.0", + ("Sekai.Mysekai.MysekaiPhotoShotAnimation", "f1b1ff8667c818bf70d56dd9b284b432"): "7.0.0", + ("Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter", "a4805d90303631a00c5a8e37e1ff5a00"): "7.0.0", ("Sekai.ColorFader", "2a6b976ee5ac7d8120ec45875b46424f"): "6.8.1", ("Sekai.CrystalShop.ShopConfirmDialogPriceParts", "2a424898f46b3aeb4201e3430b854139"): "6.8.1", ("Sekai.EventBreakTimeController", "273defc7d306842f06a7362af5082289"): "6.8.1", diff --git a/src/ui/photo_layout.py b/src/ui/photo_layout.py new file mode 100644 index 0000000..4fee021 --- /dev/null +++ b/src/ui/photo_layout.py @@ -0,0 +1,53 @@ +"""Serialized photo view references, separate from runtime photo state.""" +from ui import editor_layout, talk + + +VIEW_FIELDS = ( + "_canvasGroup", "_overlayImage", "_frameImage", "_lineCornerImageList", + "_playerHeadUpDisplay", "_switchEditModeToggle", "_avatarMotionSelector", + "_photoShotBaseImage", "_photoShotIconImage", "_photoShotButton", + "_photoShotCoverBaseImage", "_photoShotIconCoverImage", "_photoShotCoverButton", + "_callCharacterButton", "_callCharacterButtonCover", "_callCharacterButtonIcon", + "_callCharacterButtonCoverIcon", "_callCharacterButtonGraphics", "_callCharacterButtonTapEffect", + "_callCharacterWaveParticleRoot", "_iconBaseImageList", "_photoEffectIconImage", + "_openAvatarMotionSelectorButton", "_showsSelectPhotoEffectDialogButton", "_openPhotoAlbumButton", + "_dofAdjustSlider", "_tripodIconList", "_photoAlbumIcon", "_remainFilmCount", + "_remainFilmCountBg", "_dofTextImage", "_remainCountTextImage", "_callCharacterTutorialGuidePrefab", +) +VIEW_ARRAYS = {"_lineCornerImageList", "_callCharacterButtonGraphics", "_iconBaseImageList", "_tripodIconList"} +ANIMATION_FIELDS = ( + "_fxPhotoShotRoot", "_photoShotAnimator", "_photoFramePreview", "_photoCameraImages", + "_photoFramePartsImage", "_photoFilmImage", "_photoFrameShadowCanvasGroup", "_showPhotoFramePreviewEvent", +) + + +def references(reader, fields, arrays=()): + result = {} + for name in fields: + if name not in arrays: + result[name] = reader.pptr() + continue + count = reader.i32() + if count < 0 or count > reader.remaining() // 12: + raise ValueError('photo reference array over-reads object') + result[name] = [reader.pptr() for _ in range(count)] + return result + + +def decode_view(reader: talk.Reader): + return references(reader, VIEW_FIELDS, VIEW_ARRAYS) + + +def decode_animation(reader: talk.Reader): + return references(reader, ANIMATION_FIELDS, {"_photoCameraImages"}) + + +def decode_presenter(reader: talk.Reader): + return editor_layout.refs(reader, ("_photoShotView", "_selectFrameView", "_photoSaveAnimation")) + + +DECODERS = { + "Sekai.Mysekai.MysekaiPhotoShotView": decode_view, + "Sekai.Mysekai.MysekaiPhotoShotAnimation": decode_animation, + "Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter": decode_presenter, +} diff --git a/tests/test_jp_decoders.py b/tests/test_jp_decoders.py index 17dd971..a16b6b7 100644 --- a/tests/test_jp_decoders.py +++ b/tests/test_jp_decoders.py @@ -162,10 +162,10 @@ def test_700_table_holds_the_changed_classes_only(): changed; every other class is the same chain in both tables.""" table_681 = jd.decoders_for("6.8.1") table_700 = jd.decoders_for("7.0.0") - assert set(table_700) - set(table_681) == set(_VERSIONED_BODIES) - { + assert set(table_700) - set(table_681) == (set(_VERSIONED_BODIES) - { "Sekai.Mysekai.MysekaiInfoOption1Page", "Sekai.Mysekai.Inventory.CraftPreview", - } + }) | set(jd.photo_layout.DECODERS) for name, body in _VERSIONED_BODIES.items(): assert table_700[name].__name__ == body for name in set(table_681) & set(table_700): diff --git a/tests/test_s3_deployment.py b/tests/test_s3_deployment.py index 8c6ad54..c690118 100644 --- a/tests/test_s3_deployment.py +++ b/tests/test_s3_deployment.py @@ -152,6 +152,39 @@ def test_local_dry_run_never_creates_client_or_receipt(s3, tmp_path, capsys): assert set(tmp_path.iterdir()) == before +def test_translation_publication_selects_only_its_immutable_objects_or_one_pointer(s3, tmp_path, capsys): + from pipeline.text_product import build_product + from prepare_translations import prepare + source = {'schema': 'moly-text-source/1', 'region': 'jp', 'revision': 'a' * 64, + 'scripts': [{'key': 'sample', 'contents': [], 'lines': [ + {'id': 'one', 'sourceHash': 'b' * 64, 'kind': 'text', 'step': 0, 'text': 'Hello'}]}]} + overlay = {'schema': 'moly-text-overlay/1', 'sourceRegion': 'jp', 'sourceRevision': 'a' * 64, + 'locale': 'zh-CN', 'entries': [{'id': 'one', 'sourceHash': 'b' * 64, 'text': 'Welcome', 'source': 'official_cn'}]} + product = tmp_path / 'product' + for path, body in build_product(source, [overlay]).items(): + file = product / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(body) + staging = tmp_path / 'staging' + prepare(product / 'translations/jp/zh-CN', staging, object_prefix='text/', url_prefix='/moly/') + manifest = staging / 'upload-manifest.json' + arguments = cli(manifest) + arguments[arguments.index('--object-prefix') + 1] = 'text/' + args = s3.parse_args(arguments) + _, stable = s3.selection(args) + assert stable and all('/objects/' in row['path'] or '/revisions/' in row['path'] for row in stable) + assert s3.main(arguments) == 0 + assert json.loads(capsys.readouterr().out.splitlines()[-1])['cloudRequests'] == 0 + args = s3.parse_args(arguments + ['--scope', 'mutable', '--allow-mutable-update', + '--mutable-path', 'translations/jp/zh-CN/latest.json']) + _, pointers = s3.selection(args) + assert [row['path'] for row in pointers] == ['translations/jp/zh-CN/latest.json'] + doc = json.loads(manifest.read_bytes()) + doc['files'].append(row_at(tmp_path, 'snapshots/unrelated/assets/x.bin', prefix='text/')) + manifest.write_text(json.dumps(doc), encoding='utf-8') + assert_failure(s3, 'translation_manifest_contains_unrelated_object', lambda: s3.selection(args)) + + def test_s3_selection_allows_literal_object_key_delimiters(s3, tmp_path): row = row_at(tmp_path, "snapshots/cn-test/assets/atlas/balloon#40.png") manifest = manifest_at(tmp_path, [row]) diff --git a/tools/deployment/prepare_translations.py b/tools/deployment/prepare_translations.py new file mode 100644 index 0000000..ad39074 --- /dev/null +++ b/tools/deployment/prepare_translations.py @@ -0,0 +1,107 @@ +"""Prepare an independent translation publication for the existing uploaders. + +No resource snapshot is read or changed. This command performs no cloud writes. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re + +HASH = re.compile(r'[a-f0-9]{64}') + + +def prepare(publication, output, *, object_prefix, url_prefix): + publication = Path(publication).resolve() + output = Path(output).resolve() + root_bytes = (publication / 'latest.json').read_bytes() + if len(root_bytes) > 2 * 1024 * 1024: + raise ValueError('translation root exceeds limit') + root = json.loads(root_bytes) + if (root.get('schema') != 'moly-text-publication/1' or root.get('region') not in {'jp', 'cn', 'tw', 'en', 'kr'} + or root.get('locale') not in {'zh-CN', 'zh-TW', 'en-US', 'ja-JP', 'ko-KR'} + or not HASH.fullmatch(root.get('revision', '')) or root.get('routing') != 'sha256-utf8-prefix-2' + or not isinstance(root.get('buckets'), dict) or len(root['buckets']) > 256 + or len(root.get('searchBuckets', {})) > 16 or root.get('fonts')): + raise ValueError('invalid translation publication identity') + base = f"translations/{root['region']}/{root['locale']}/" + files = {'latest.json': root_bytes, f"revisions/{root['revision']}.json": root_bytes} + + def read(ref): + if (not isinstance(ref, dict) or not HASH.fullmatch(ref.get('sha256', '')) + or ref.get('path') != f"objects/{ref['sha256']}.json" + or type(ref.get('bytes')) is not int or not 0 < ref['bytes'] <= 2 * 1024 * 1024): + raise ValueError('invalid translation object reference') + path = (publication / ref['path']).resolve() + if not path.is_relative_to(publication): + raise ValueError('translation path escaped publication') + data = path.read_bytes() + if len(data) != ref['bytes'] or hashlib.sha256(data).hexdigest() != ref['sha256']: + raise ValueError('translation object integrity mismatch') + files[ref['path']] = data + return json.loads(data) + + for prefix, ref in root['buckets'].items(): + if not re.fullmatch('[a-f0-9]{2}', prefix): + raise ValueError('invalid translation route prefix') + routes = read(ref) + if routes.get('schema') != 'moly-text-translation-routes/1': + raise ValueError('invalid translation routes') + for key, entry in routes['scripts'].items(): + if hashlib.sha256(key.encode()).hexdigest()[:2] != prefix: + raise ValueError('translation script in wrong bucket') + overlay = read(entry) + if (overlay.get('schema') != 'moly-text-overlay/1' or overlay.get('sourceRegion') != root['region'] + or overlay.get('locale') != root['locale']): + raise ValueError('translation overlay scope mismatch') + for prefix, ref in root.get('searchBuckets', {}).items(): + search = read(ref) + if (not re.fullmatch('[a-f0-9]', prefix) or search.get('schema') != 'moly-text-translation-search/1' + or any(hashlib.sha256(key.encode()).hexdigest()[:1] != prefix for key in search['scripts'])): + raise ValueError('invalid translation search shard') + if (publication / f"revisions/{root['revision']}.json").read_bytes() != root_bytes: + raise ValueError('archived publication differs from latest') + if (not object_prefix.endswith('/') or object_prefix.startswith('/') or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for prefix in [object_prefix, url_prefix] for part in prefix.split('/')) + or any(char in object_prefix + url_prefix for char in '\\%?#:')): + raise ValueError('publication prefixes must be directories') + output.mkdir(parents=True, exist_ok=True) + rows = [] + for name, data in sorted(files.items()): + logical = base + name + target = output / 'staging' / logical + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != data: + raise ValueError('staging bytes differ; choose a fresh output directory') + if not target.exists(): + target.write_bytes(data) + sha = hashlib.sha256(data).hexdigest() + rows.append({'path': logical, 'urlPath': url_prefix + logical, 'objectKey': object_prefix + logical, + 'source': str(target), 'sourceRaw': str(target), 'staging': str(target), + 'sha256': sha, 'decodedSha256': sha, 'bytes': len(data), 'decodedBytes': len(data), + 'contentEncoding': 'identity', 'contentType': 'application/json; charset=utf-8', + 'cacheControl': 'public, max-age=15, must-revalidate' if name == 'latest.json' + else 'public, max-age=31536000, immutable'}) + result = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, + 'urlPrefix': url_prefix, 'snapshots': [], 'translation': {'path': base + 'latest.json', + 'revision': root['revision'], 'sha256': hashlib.sha256(root_bytes).hexdigest()}, + 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows)}, 'files': rows} + (output / 'upload-manifest.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--publication', required=True, type=Path, help='Locale directory containing latest.json') + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--object-prefix', default='sekai-extra-assets/') + parser.add_argument('--url-prefix', default='/moly/') + args = parser.parse_args() + result = prepare(args.publication, args.output, object_prefix=args.object_prefix, url_prefix=args.url_prefix) + print(json.dumps({'translation': result['translation'], 'totals': result['totals']})) + + +if __name__ == '__main__': + main() diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 6787689..f73f938 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -25,6 +25,13 @@ ENDPOINT = 'https://oss-cn-shanghai.aliyuncs.com' PREFIX = 'moly/alpha/' MUTABLE = {'manifest.json', 'cache-worker.mjs', 'game/manifest.json'} +TRANSLATION_ROOT = re.compile(r'translations/(jp|cn|tw|en|kr)/(zh-CN|zh-TW|en-US|ja-JP|ko-KR)/latest\.json') + + +def mutable_path(value): + if value not in MUTABLE and not TRANSLATION_ROOT.fullmatch(value): + raise argparse.ArgumentTypeError('expected a known publication pointer') + return value IDENTIFIER = re.compile(r'^[a-z0-9][a-z0-9._-]{0,95}$') SHA256 = re.compile(r'^[a-f0-9]{64}$') @@ -144,7 +151,7 @@ def parse_args(): p.add_argument('--manifest', type=Path, required=True) p.add_argument('--scope', choices=['stable', 'release', 'descriptors', 'mutable'], default='stable') p.add_argument('--release-id', help='Required explicit release ID for --scope release') - p.add_argument('--mutable-path', choices=sorted(MUTABLE), action='append', default=[]) + p.add_argument('--mutable-path', type=mutable_path, action='append', default=[]) p.add_argument('--allow-mutable-update', action='store_true') p.add_argument('--expect-previous-sha256', help='Required to replace an existing mutable object; one mutable path per run') p.add_argument('--threads', type=int, default=16, choices=range(1, 17), metavar='1..16') @@ -169,7 +176,17 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) if doc.get('deliveryMode', 'fixed') != 'fixed': raise SafeFailure('unsupported_delivery_mode') snapshots = set(doc.get('snapshots', [])) - if not snapshots or any(not IDENTIFIER.fullmatch(s) for s in snapshots): + translation = doc.get('translation') + if translation is not None: + if (not isinstance(translation, dict) or snapshots + or not TRANSLATION_ROOT.fullmatch(translation.get('path', '')) + or not SHA256.fullmatch(translation.get('revision', '')) + or not SHA256.fullmatch(translation.get('sha256', ''))): + raise SafeFailure('invalid_translation_publication') + translation_base = translation['path'].removesuffix('latest.json') + else: + translation_base = None + if (not snapshots and translation is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): raise SafeFailure('invalid_snapshot_identities') if args.scope == 'release' and (args.release_id != doc.get('release') or not IDENTIFIER.fullmatch(args.release_id or '')): raise SafeFailure('explicit_release_id_must_match_manifest') @@ -187,6 +204,9 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected, seen = [], set() for row in doc.get('files', []): logical = row.get('path', '') + if translation_base and (not logical.startswith(translation_base) or not re.fullmatch( + r'(?:latest\.json|(?:objects|revisions)/[a-f0-9]{64}\.json)', logical[len(translation_base):])): + raise SafeFailure('translation_manifest_contains_unrelated_object') parts = PurePosixPath(logical).parts # This is an object key, not a URL. S3 permits a literal '#'; URL # consumers must percent-encode that segment when constructing a @@ -220,8 +240,9 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) is_snapshot = len(parts) >= 3 and parts[0] == 'snapshots' and parts[1] in snapshots # The shared content-addressed stores are immutable like snapshot payloads. shared_store = len(parts) >= 2 and parts[0] in {'asset-store', 'catalog-store'} + translation_object = translation_base and logical.startswith(translation_base) and logical != translation['path'] choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog', 'text'} and len(parts) >= 4) - or shared_store)) + or shared_store or translation_object)) or (args.scope == 'descriptors' and is_snapshot and len(parts) == 3 and parts[2] in {'snapshot.json', 'provenance.json'}) or (args.scope == 'release' and len(parts) >= 3 and parts[:2] == ('releases', args.release_id)) or (args.scope == 'mutable' and logical in args.mutable_path)) @@ -232,6 +253,17 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected = selected[:args.limit] if not selected: raise SafeFailure('no_objects_selected') + if translation is not None: + roots = [row for row in doc['files'] if row['path'] == translation['path']] + if len(roots) != 1 or roots[0]['decodedSha256'] != translation['sha256']: + raise SafeFailure('translation_pointer_identity_mismatch') + raw = Path(roots[0]['sourceRaw']).read_bytes() + root = json.loads(raw) + parts = translation['path'].split('/') + if (hashlib.sha256(raw).hexdigest() != translation['sha256'] + or root.get('schema') != 'moly-text-publication/1' or root.get('region') != parts[1] + or root.get('locale') != parts[2] or root.get('revision') != translation['revision']): + raise SafeFailure('local_translation_pointer_changed') # Publishing descriptions requires the actual production descriptor bytes. for row in selected: if row['path'] in {'manifest.json', 'game/manifest.json'} or row['path'].endswith('/snapshot.json'): @@ -291,7 +323,7 @@ def put_one(pool, row, args, stop): # Legacy mutable pointers predate uploader metadata. Verify their # actual stored bytes, bounded to this tiny control-plane surface, # instead of treating absent metadata as proof of their identity. - if previous is None and args.expect_previous_sha256 and key in {PREFIX + name for name in MUTABLE}: + if previous is None and args.expect_previous_sha256 and is_mutable: response = pool.get().get_object(key) try: body = response.read(1048577) diff --git a/tools/deployment/upload_s3.py b/tools/deployment/upload_s3.py index e9e9b24..69bc132 100644 --- a/tools/deployment/upload_s3.py +++ b/tools/deployment/upload_s3.py @@ -607,7 +607,7 @@ def parse_args(argv=None): parser.add_argument('--profile', help='Optional local AWS SDK profile; otherwise the standard credential chain') parser.add_argument('--scope', choices=['stable', 'release', 'descriptors', 'mutable'], default='stable') parser.add_argument('--release-id') - parser.add_argument('--mutable-path', choices=sorted(common.MUTABLE), action='append', default=[]) + parser.add_argument('--mutable-path', type=common.mutable_path, action='append', default=[]) parser.add_argument('--allow-mutable-update', action='store_true') parser.add_argument('--expect-previous-sha256') parser.add_argument('--limit', type=int) From d4b1dc2294b7b3570e184a1aba49e96d7cd5dc61 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:26:19 +0800 Subject: [PATCH 03/15] feat(hfsm): export tutorial and photo runtime inputs --- src/chara/characters.py | 33 ++++++++++++++ src/chara/talks.py | 26 +++++++++++ src/chara/text_catalog.py | 2 + src/phenomena/audio_corpus.py | 4 +- src/pipeline/stages.py | 4 +- src/ui/photo_layout.py | 67 +++++++++++++++++++++++++++++ src/ui/prefab_layout.py | 16 +++++++ tests/test_birthday_party_talks.py | 4 +- tools/migrate_canonical_metadata.py | 4 +- 9 files changed, 154 insertions(+), 6 deletions(-) diff --git a/src/chara/characters.py b/src/chara/characters.py index 95d8fb2..0e59752 100644 --- a/src/chara/characters.py +++ b/src/chara/characters.py @@ -319,8 +319,16 @@ def read_renderers(assets, tr_index): raise KeyError("renderer bone outside the prefab tree") bones.append(bi) go = gos.get(go_pid) + bounds = tt.get('m_AABB') + if bounds is None: + raise ValueError('skinned renderer has no authored bounds') + root_bone = tr_index.get(tt.get('m_RootBone', {}).get('m_PathID', 0)) out.append({"mesh": mesh_obj, "materialNames": mats, "materialObjs": mat_objs, "bones": bones, "node": node, + "bounds": {"node": node, "rootBone": root_bone, + "center": list(unity_to_gltf_pos([bounds['m_Center'][k] for k in ('x', 'y', 'z')])), + "extents": [bounds['m_Extent'][k] for k in ('x', 'y', 'z')], + "updateWhenOffscreen": bool(tt.get('m_UpdateWhenOffscreen', False))}, "rendererName": go["m_Name"] if go else ""}) return out @@ -483,6 +491,29 @@ def chan(node, path, data, dim): return report +def head_ik_axes(avatar, nodes): + """Bind humanoid look-at axes to the emitted skeleton in canonical space.""" + human = avatar['human'] + def bone(slot): + index = human['humanBoneIndex'][slot] + if index < 0: + return None + source = human['nodes'][index] + matches = [i for i, node in enumerate(nodes) if node['name'] == source['name']] + if len(matches) != 1 or source['axesId'] < 0: + raise ValueError(f'look-at bone has no unique skeleton/axes binding: {source["name"]}') + axes = human['axes'][source['axesId']] + def quaternion(value): + # Reflection of X preserves quaternion X/W and negates Y/Z. + return [value[0], -value[1], -value[2], value[3]] + return {'node': matches[0], 'name': source['name'], + 'preQ': quaternion(axes['preQ']), 'postQ': quaternion(axes['postQ'])} + head, neck = bone(11), bone(10) + if head is None: + raise ValueError('humanoid avatar has no head binding') + return {**head, 'neck': neck} + + def extract_character(bundle, out_dir, name, sampled=None, aux=(), unit_id=None, facial=None, atlas_cell=DEFAULT_ATLAS_CELL, with_cloth=True): @@ -788,6 +819,8 @@ def wm(i): "mouthAtlas": _atlas_doc(mouth_tex.m_Width, mouth_tex.m_Height, atlas_cell), "cloth": cloth_rig, "anchors": read_anchors(assets, tr_index, nodes), + "headIk": head_ik_axes(rig.doc, nodes), + "rendererBounds": [{**renderer['bounds'], 'name': renderer['rendererName']} for renderer in renderers], "textures": [tex_paths[n] for n in sorted(pngs)], # materials[i].name's contract is "the glTF material's own name", # matching the site domain's convention (there it holds the real diff --git a/src/chara/talks.py b/src/chara/talks.py index 6e3322b..e835230 100644 --- a/src/chara/talks.py +++ b/src/chara/talks.py @@ -773,6 +773,29 @@ def birthday_talks(master, assets, tables, scalars): return block, dict(report, status="present") +def tutorial_talks(master, assets, tables, scalars): + """Tutorial IDs select authored scripts; they do not enter the talk lottery.""" + try: + rows = master.table('mysekaiTutorialTalks') + except MissingTable: + return {'status': 'absent', 'rows': [], 'talks': {}}, {'status': 'absent', 'scripts': 0} + scripts = {} + for row in rows: + lua = row['lua'] + if lua in scripts: + continue + name = lua if lua.endswith('.lua') else lua + '.lua' + script = _asset_named(assets, name) + if script is None: + raise LookupError(f'tutorial script asset not found: {name}') + parsed = parse_script(script, tables, scalars, script_name=name) + scripts[lua] = {'voices': parsed['voices'], 'steps': parsed['steps'], + 'voiceBank': 'mysekai/talk/tutorial/voice/' + lua} + return {'status': 'complete', 'rows': rows, 'talks': scripts}, { + 'status': 'complete', 'scripts': len(scripts), 'rows': len(rows), + 'steps': sum(len(script['steps']) for script in scripts.values())} + + def _semantics(source): return { "selection": ( @@ -919,6 +942,8 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, from chara.common_talks import common_talks common, common_report = common_talks(master, assets, tables, scalars) doc["commonSomeCharacter"] = common + tutorial, tutorial_report = tutorial_talks(master, assets, tables, scalars) + doc['tutorial'] = tutorial os.makedirs(os.path.dirname(os.path.abspath(out_path)) or ".", exist_ok=True) with open(out_path, "w", encoding="utf-8", newline="\n") as handle: json.dump(doc, handle, ensure_ascii=False, indent=1, allow_nan=False) @@ -930,6 +955,7 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, "constantTables": {key: len(value) for key, value in tables.items()}, "birthdayParty": birthday_report, "commonSomeCharacter": common_report, + "tutorial": tutorial_report, } diff --git a/src/chara/text_catalog.py b/src/chara/text_catalog.py index ddd88d5..a4afe35 100644 --- a/src/chara/text_catalog.py +++ b/src/chara/text_catalog.py @@ -30,6 +30,8 @@ def corpus_rows(document): yield "common", row, row["unitIds"] for lua, row in (document.get("birthdayParty") or {}).get("talks", {}).items(): yield "birthday", {**row, "lua": lua, "talkId": lua}, [] + for lua, row in (document.get("tutorial") or {}).get("talks", {}).items(): + yield "tutorial", {**row, "lua": lua, "talkId": lua}, [] def _display_rows(row): diff --git a/src/phenomena/audio_corpus.py b/src/phenomena/audio_corpus.py index c72af9b..ed34ae0 100644 --- a/src/phenomena/audio_corpus.py +++ b/src/phenomena/audio_corpus.py @@ -123,6 +123,8 @@ def _corpus_talks(document): yield from unit.get("talks") or [] yield from document.get("talks") or [] yield from (document.get("commonSomeCharacter") or {}).get("talks") or [] + for lua, row in ((document.get("tutorial") or {}).get("talks") or {}).items(): + yield {**row, 'lua': lua} def voice_requests(document): @@ -143,7 +145,7 @@ def voice_requests(document): # stripping a fixed number of underscore fields does not. stem = script if isinstance(script, str) and cue.startswith( CUE_PREFIX + script + "_") else voice_stem(cue) - requests.setdefault(TALK_VOICE + stem, set()).add(cue) + requests.setdefault(talk.get('voiceBank') or TALK_VOICE + stem, set()).add(cue) # A party script's cues live in the bank the talk engine loads under the # Birthday context, not in the plain one, and the talk document names that # bank per script; it is read from there rather than assumed from a prefix. diff --git a/src/pipeline/stages.py b/src/pipeline/stages.py index 446e680..46946fb 100644 --- a/src/pipeline/stages.py +++ b/src/pipeline/stages.py @@ -52,7 +52,7 @@ def talk_task(master, bundle, lib, *, family="general", unity_version="2022.3.62 else: # Optional tables are part of the input identity when present. Their # absence is represented by the table list, not a made-up empty file. - tables += [name for name in ("birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks") + tables += [name for name in ("birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", "mysekaiTutorialTalks") if (Path(master) / (name + ".json")).is_file()] inputs += [file_input("master/" + name, Path(master) / (name + ".json")) for name in tables] return task("talks.extract", family, inputs, family=family, unityVersion=unity_version) @@ -100,7 +100,7 @@ def validate_installed(value): elif name == "talks.extract": if value["params"]["family"] not in {"general", "fixture"}: raise ValueError("unknown conversation family") - permitted = set(TALK_TABLES) | {"birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", + permitted = set(TALK_TABLES) | {"birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", "mysekaiTutorialTalks", "mysekaiFixtures", "mysekaiCharacterTalkNoTalkMysekaiFixtureActions"} if not {"bundle", "lib"} <= roles or any( role not in {"bundle", "lib"} and role not in {"master/" + t for t in permitted} diff --git a/src/ui/photo_layout.py b/src/ui/photo_layout.py index 4fee021..0cbc26b 100644 --- a/src/ui/photo_layout.py +++ b/src/ui/photo_layout.py @@ -1,5 +1,72 @@ """Serialized photo view references, separate from runtime photo state.""" from ui import editor_layout, talk +from pathlib import Path +import hashlib +import UnityPy + +PACKAGE = "mysekai__ui__mysekai_photo_shot" +IMAGES = ( + "overlay_photoshot_screen", "frame_PhotoShot", "line_cornerguide_photoshot", + "btn_photoshot_gr_h180", "icon_callout_wh", "icon_camera_wh_h80", "btn_circle120_lgr", + "icon_cameraEffect_h56", "icon_tripod_wh_h44", "icon_photoalbum_wh_h42", + "txt_photoshot_blur", "txt_photoshot_filmCount", +) + + +def package_images(data, out): + """The view loads images by these authored names when it opens.""" + env = UnityPy.load(data) + sprites = {} + for obj in env.objects: + if obj.type.name != 'Sprite': + continue + tree = obj.read_typetree() + # A bundle also includes common-atlas sprite aliases whose atlas data + # is supplied elsewhere. The photo view's own standalone sprite is + # the one with a concrete unpacked texture and no atlas binding. + if tree.get('m_SpriteAtlas', {}).get('m_PathID') or tree['m_RD']['settingsRaw'] & 1: + continue + name = tree['m_Name'] + if name in sprites: + raise ValueError(f'duplicate unpacked photo sprite {name}') + sprites[name] = obj + textures = {obj.read_typetree()["m_Name"]: obj for obj in env.objects if obj.type.name == "Texture2D"} + paths, metadata = {}, {} + output = Path(out) + (output / 'textures').mkdir(parents=True, exist_ok=True) + for name in IMAGES: + sprite = sprites.get(name) + obj = sprite or textures.get(name) + if obj is None: + raise ValueError(f'photo package is missing authored image {name}') + relative = f'textures/photo-{name}.png' + image = obj.read() + if sprite is not None and not image.m_SpriteAtlas: + # These unpacked sprites retain a CommonAtlas tag even though the + # package's atlas is empty. Their own render-data texture is the + # authored source; do not resolve a tag to an unrelated atlas. + if image.m_RD.settingsRaw & 1 or not image.m_RD.texture: + raise ValueError(f'photo sprite has no unpacked texture: {name}') + image.m_AtlasTags = [] + image.image.save(output / relative) + paths[name] = relative + if sprite is not None: + tree = sprite.read_typetree() + rect, rd = tree['m_Rect'], tree['m_RD'] + metadata[name] = { + 'name': name, 'image': relative, 'package': PACKAGE, 'pathId': obj.path_id, + 'rect': [rect[k] for k in ('x', 'y', 'width', 'height')], + 'rectSize': [rect['width'], rect['height']], + 'border': [tree['m_Border'][k] for k in ('x', 'y', 'z', 'w')], + 'pixelsPerUnit': tree['m_PixelsToUnits'], + 'pivot': [tree['m_Pivot']['x'], tree['m_Pivot']['y']], + 'textureRect': [rd['textureRect'][k] for k in ('x', 'y', 'width', 'height')], + 'textureRectOffset': [rd['textureRectOffset']['x'], rd['textureRectOffset']['y']], + 'packingSettingsRaw': rd['settingsRaw'], + 'downscaleMultiplier': rd.get('downscaleMultiplier', 1.0), + } + return paths, metadata, {'package': PACKAGE, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), + 'images': len(paths), 'sprites': len(metadata)} VIEW_FIELDS = ( diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index 1b3eccb..4440358 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -3050,6 +3050,14 @@ def extract_layout_jp(apks, out_dir, *, client_version, master_dir, prefabs, sitemap_package = Path(bundles_root) / SITEMAP_TEXTURE_PACKAGE if not sitemap_package.is_file(): raise FileNotFoundError(f"UI texture package missing: {sitemap_package.name}") + from ui import photo_layout + photo_package = None + if "ScreenLayerMysekaiPhotoShot" in selected: + if bundles_root is None: + raise ValueError("photo-shot UI needs its decrypted texture package") + photo_package = Path(bundles_root) / photo_layout.PACKAGE + if not photo_package.is_file(): + raise FileNotFoundError(f"UI texture package missing: {photo_package.name}") bgm_player_package = None if "ScreenLayerMysekaiBGMSelect" in selected: if bundles_root is None: @@ -3362,6 +3370,14 @@ def digest(path): runtime_textures.update(textures) runtime_sprites.update(sprites) texture_sources.append(source) + if photo_package is not None: + textures, sprites, source = photo_layout.package_images(photo_package.read_bytes(), out) + for alias in textures: + if alias in runtime_textures: + raise ValueError(f"UI runtime texture alias collision: {alias}") + runtime_textures.update(textures) + runtime_sprites.update(sprites) + texture_sources.append(source) for item in plan: prefab, family = item["prefab"], item["family"] file_name, root_pid = item["target"] diff --git a/tests/test_birthday_party_talks.py b/tests/test_birthday_party_talks.py index 101d332..5ad03a3 100644 --- a/tests/test_birthday_party_talks.py +++ b/tests/test_birthday_party_talks.py @@ -127,7 +127,7 @@ def test_party_and_common_blocks_preserve_the_ordinary_corpus_keys(tmp_path, mon talks.extract_talks(master, "synthetic.bundle", out) document = json.loads(out.read_text(encoding="utf-8")) assert list(document) == ["version", "semantics", "units", "summary", - "birthdayParty", "commonSomeCharacter"] + "birthdayParty", "commonSomeCharacter", "tutorial"] assert document["birthdayParty"]["semantics"]["table"].startswith( "birthdayPartyScenarios") @@ -175,7 +175,7 @@ def test_attach_leaves_every_other_key_alone(tmp_path, monkeypatch): assert after["birthdayParty"]["status"] == "present" assert after["birthdayParty"]["talks"]["party_001"]["voices"] == ["voice_party_001_01_006"] assert report["preservedKeys"] == ["version", "semantics", "units", "summary", - "commonSomeCharacter"] + "commonSomeCharacter", "tutorial"] # Byte for byte: with the block taken back out, the attached document is # the corpus the extraction wrote, minus that run's own absent block. del after["birthdayParty"] diff --git a/tools/migrate_canonical_metadata.py b/tools/migrate_canonical_metadata.py index cd15c22..a73ab20 100644 --- a/tools/migrate_canonical_metadata.py +++ b/tools/migrate_canonical_metadata.py @@ -25,7 +25,9 @@ # They keep the guard usable in source archives and installed source copies. # Update them only after reviewing the coordinate-writing code. REVIEWED_PRODUCERS = { - 'src/chara/characters.py': 'e18e112c332b31c26110e73aaacfc90911c5124f28782fff8e2da0bcd006a25f', + # Adds sidecar look-at axes and renderer bounds using the existing X + # reflection. Mesh vertices, animation, skinning and GLB writes are unchanged. + 'src/chara/characters.py': 'a7f8e51574dac542f0bc4ea07077736f929eb08634ef385149bcd4269323461d', 'src/chara/motion_library.py': '3ca5702e6ff3d02ee065c32dc5a2e1fe24716999976c135ffa77e52a3b26465c', 'src/chara/player_avatar.py': 'f10f702f6fc3b58f23c1f1a5a388bbe94789e27fec7d93d2b8316abf13a012e1', 'src/chara/avatar_parts.py': '74a42b5519bae8b645850e0281420c707ebac33fa08df3f5bd6326a4dcf89fc3', From 9ce6038d24b644dcdddfa0cb102bfd7cf9786e1c Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:02:48 +0800 Subject: [PATCH 04/15] feat(runtime): export tutorial poses and scoped photo effects --- docs/jp-runtime.md | 17 +++++++ src/chara/tutorial.py | 109 ++++++++++++++++++++++++++++++++++++++++ src/cli/legacy.py | 13 +++++ src/pack/groups.py | 2 +- src/ui/photo_layout.py | 45 +++++++++++++++++ src/ui/prefab_layout.py | 2 + 6 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 src/chara/tutorial.py diff --git a/docs/jp-runtime.md b/docs/jp-runtime.md index fa27331..022e72b 100644 --- a/docs/jp-runtime.md +++ b/docs/jp-runtime.md @@ -1,5 +1,22 @@ # Japanese runtime resources +NPC tutorial poses are exported from the client Resources index with the same +canonical position/quaternion basis as characters and sites: + +```sh +moly export tutorial-placements --apk base.apk --apk asset-pack.apk \ + --client-version 7.0.0 --out-dir runtime/tutorial \ + --processor-input source/tutorial-processor.json +``` + +The optional processor input contains source-qualified introductory poses that +are defined by client code rather than serialized placements. Its identity and +canonical coordinates are checked before writing. Section progress and other +server state remain inputs to the runtime; they are not invented by extraction. +The photo-shot UI export also includes its two dynamically loaded call effects, +each restricted to its own referenced objects so identical internal node names +from neighboring prefabs cannot collide. + The built-in JP commands follow the public asset-discovery protocol documented by `sssekai`. Install the optional protocol dependencies with `pip install -e '.[jp]'`. Discovery and downloads happen only when requested. diff --git a/src/chara/tutorial.py b/src/chara/tutorial.py new file mode 100644 index 0000000..fee56c5 --- /dev/null +++ b/src/chara/tutorial.py @@ -0,0 +1,109 @@ +"""NPC tutorial placements from the client Resources index.""" +import hashlib +import json +import math +from pathlib import Path + +from core.gltf import coordinate_contract, reflect_position, reflect_quaternion +from core.jsonio import write_json +from ui import talk +from ui.player_source import ApkPlayerData, container_target, resource_container, unity_version + +PLACEMENT_CLASS = 'Sekai.Mysekai.MysekaiTutorialCharacterPlacementData' +PLACEMENT_TYPE = '3c921a0c7b9d90b022e9c5428407c181' +PLACEMENT_PREFIX = 'mysekai/tutorial/placement/' + + +def decode_placements(reader): + count = reader.i32() + if count < 0 or count > reader.remaining() // 32: + raise ValueError('tutorial placement count exceeds its serialized object') + return {'datas': [{'unitCharacterId': reader.i32(), 'position': reader.vec3(), + 'rotation': reader.vec4()} for _ in range(count)]} + + +def processor_configuration(path, *, client_version): + """Validate separately supplied client constants; never invent server state.""" + raw = Path(path).read_bytes() + if len(raw) > 65536: + raise ValueError('tutorial processor configuration exceeds its size limit') + value = json.loads(raw) + if not isinstance(value, dict) or set(value) != { + 'schema', 'region', 'clientVersion', 'coordinateContract', 'initialOffstage', + 'initialPlayer', 'afterIntroNpc', 'afterTalkPlayerRotation', + } or value['schema'] != 'moly-tutorial-processor/1' or value['region'] != 'jp' or value['clientVersion'] != client_version: + raise ValueError('tutorial processor configuration has a different source identity') + if value['coordinateContract'] != coordinate_contract()['space']: + raise ValueError('tutorial processor coordinates must already be canonical') + + def vector(data, width): + if not isinstance(data, list) or len(data) != width or any( + type(v) not in {int, float} or not math.isfinite(v) for v in data + ): + raise ValueError('invalid tutorial processor pose') + if width == 4 and not math.isclose(sum(v * v for v in data), 1, abs_tol=1e-4): + raise ValueError('invalid tutorial processor rotation') + + vector(value['initialOffstage'], 3) + vector(value['afterTalkPlayerRotation'], 4) + for name in ['initialPlayer', 'afterIntroNpc']: + pose = value[name] + if not isinstance(pose, dict) or set(pose) != {'position', 'rotation'}: + raise ValueError('invalid tutorial processor transform') + vector(pose['position'], 3) + vector(pose['rotation'], 4) + return value + + +def export_placements(apks, out_dir, *, client_version, processor_input=None): + if client_version != '7.0.0': + raise ValueError('tutorial placement extraction requires a supported source version') + processor = None if processor_input is None else processor_configuration(processor_input, client_version=client_version) + data = ApkPlayerData(apks) + player = data.environment() + player.add(['globalgamemanagers'], follow=False) + manager, container = resource_container(player.env) + selected = [] + for key, pointers in sorted(container.items()): + if not key.startswith(PLACEMENT_PREFIX): + continue + if len(pointers) != 1: + raise ValueError(f'ambiguous tutorial placement resource {key}') + selected.append((key, container_target(player.env, manager, pointers[0]))) + if not selected: + raise ValueError('source has no tutorial placement resources') + player.add([target[0] for _, target in selected]) + if player.missing: + raise ValueError(f'tutorial placements have missing source files: {player.missing}') + scripts = talk.build_monoscript_index(player.env) + placements, sources = {}, [] + for key, (file_name, path_id) in selected: + obj = player.env.files[file_name].objects[path_id] + if talk.resolve_script_class(obj, scripts) != PLACEMENT_CLASS or talk.serialized_type_hash(obj) != PLACEMENT_TYPE: + raise ValueError(f'unrecognized tutorial placement layout: {key}') + decoded = talk.decode_object(player.env, obj, scripts, decoders={PLACEMENT_CLASS: decode_placements}) + if decoded['residual']: + raise ValueError(f'tutorial placement has unread serialized bytes: {key}') + rows = decoded['fields']['datas'] + ids = [row['unitCharacterId'] for row in rows] + if any(value <= 0 for value in ids) or len(set(ids)) != len(ids): + raise ValueError(f'invalid tutorial participant identities: {key}') + for row in rows: + if not all(math.isfinite(value) for value in row['position'] + row['rotation']): + raise ValueError(f'non-finite tutorial pose: {key}') + if not math.isclose(sum(v * v for v in row['rotation']), 1, abs_tol=1e-4): + raise ValueError(f'invalid tutorial rotation: {key}') + placements[key.removeprefix(PLACEMENT_PREFIX)] = [ + {'unitCharacterId': row['unitCharacterId'], + 'position': reflect_position(row['position']), + 'rotation': reflect_quaternion(row['rotation'])} for row in rows] + sources.append({'resource': key, 'serializedFile': file_name, 'pathId': path_id, + 'sourceSha256': hashlib.sha256(obj.get_raw_data()).hexdigest(), + 'typeHash': PLACEMENT_TYPE, 'rows': len(rows), 'residual': 0}) + result = {'schema': 'moly-tutorial-placements/1', 'region': 'jp', 'clientVersion': client_version, + 'unityVersion': unity_version(player.env), 'coordinates': coordinate_contract(), + 'placements': placements, 'sources': sources} + write_json(Path(out_dir) / 'placements.json', result) + if processor is not None: + write_json(Path(out_dir) / 'processor.json', processor) + return result diff --git a/src/cli/legacy.py b/src/cli/legacy.py index 9371267..1b471f4 100644 --- a/src/cli/legacy.py +++ b/src/cli/legacy.py @@ -671,6 +671,11 @@ def main(argv=None): fixture_animations.add_argument("--plan", action="store_true") capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") capabilities.add_argument("--asset-root", required=True) + tutorial = sub.add_parser("tutorial-placements", help="export source NPC tutorial placement poses") + tutorial.add_argument("--apk", action="append", required=True) + tutorial.add_argument("--client-version", required=True) + tutorial.add_argument("--out-dir", required=True) + tutorial.add_argument("--processor-input", help="optional source-qualified canonical intro pose configuration") lay = sub.add_parser("ui-layout", help="extract the RectTransform hierarchy of the UI " "prefabs (screen layers, dialogs) the player data " @@ -1254,6 +1259,14 @@ def main(argv=None): print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) return 0 + if args.cmd == "tutorial-placements": + from chara.tutorial import export_placements + result = export_placements(args.apk, args.out_dir, client_version=args.client_version, + processor_input=args.processor_input) + print(json.dumps({"placements": len(result['placements']), + "participants": sum(len(rows) for rows in result['placements'].values())})) + return 0 + if args.cmd == "ui-layout" and args.region == "jp": from ui.prefab_layout import extract_layout_jp if not (args.apk and args.client_version and args.master and args.prefab): diff --git a/src/pack/groups.py b/src/pack/groups.py index 250c89d..b671393 100644 --- a/src/pack/groups.py +++ b/src/pack/groups.py @@ -66,7 +66,7 @@ "effect-resources", "graphics-config", "layout-grid", "site-action-effects", "site-common-action-effects", "site-harvest-object-effects", "site-home-effects", "fixture-put-effects", "site-sound-wave", "site-timeline", - "honor", "honor_frame", "material-thumbnails", "text", + "honor", "honor_frame", "material-thumbnails", "text", "tutorial", ) FORMATS = {".json", ".glb", ".gltf", ".png", ".jpg", ".jpeg", ".webp", ".ktx2", ".rgba8", ".ogg", ".wav", ".mp3", ".bin"} BUILD_EVIDENCE_FILES = {"export-report.json", "export-plan.json", "export-progress.json", diff --git a/src/ui/photo_layout.py b/src/ui/photo_layout.py index 0cbc26b..65cc2a7 100644 --- a/src/ui/photo_layout.py +++ b/src/ui/photo_layout.py @@ -69,6 +69,51 @@ def package_images(data, out): 'images': len(paths), 'sprites': len(metadata)} +def package_effects(bundle, bundle_root, out): + """Export each dynamically loaded effect with only its reachable objects. + + These prefabs share node names inside one package. Exporting all roots + together makes a UI particle's node binding ambiguous and includes unrelated + atlas aliases. The existing prefab view follows the selected root's actual + component, material and texture references before the normal scene export. + """ + from core.assets.packages import PackageStore + from core.jsonio import write_json + from core.player_resources import prefab_view + from sites.scenes import PackageExtract + + store = PackageStore([bundle], bundle_root) + store.load_dependencies([Path(bundle).name]) + package = store.package(Path(bundle).name) + records = [] + for role, prefab in ( + ('call-button', 'fx_mysekai_photo_shot_call_character_pt'), + ('call-wave', 'fx_mysekai_photo_shot_call_character_wave'), + ): + targets = [(record, path_id) for name, record, path_id in package.contents + if name == prefab + '.prefab'] + if len(targets) != 1: + raise ValueError(f'photo effect {prefab} needs exactly one prefab root') + view, report = prefab_view(store, targets, f'photo-shot:{role}') + if report['unreadable'] or report.get('unfollowedArchives'): + raise ValueError(f'photo effect {prefab} has an incomplete object closure: {report}') + store.adopt(view) + directory = Path(out) / 'photo-shot' / role + document = PackageExtract(store, view.name, directory, role, + classification=('prefab-package', role)).run() + primary = [root for root in document['roots'] if root['primary']] + if len(primary) != 1 or primary[0]['name'] != prefab or document['unsupported']: + raise ValueError(f'photo effect {prefab} is incomplete: {document["unsupported"]}') + document['source'] = {'package': store.source_receipt(Path(bundle).name), + 'prefab': prefab, 'view': report} + relative = f'photo-shot/{role}/{role}.json' + write_json(Path(out) / relative, document) + records.append({'document': relative, 'kind': 'photo-particle', + 'documentSha256': hashlib.sha256((Path(out) / relative).read_bytes()).hexdigest(), + 'source': document['source'], 'particles': len(document['particles'])}) + return records + + VIEW_FIELDS = ( "_canvasGroup", "_overlayImage", "_frameImage", "_lineCornerImageList", "_playerHeadUpDisplay", "_switchEditModeToggle", "_avatarMotionSelector", diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index 4440358..ef0172d 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -3378,6 +3378,8 @@ def digest(path): runtime_textures.update(textures) runtime_sprites.update(sprites) texture_sources.append(source) + for effect in photo_layout.package_effects(photo_package, bundles_root, out): + rows.append({**effect, **identity}) for item in plan: prefab, family = item["prefab"], item["family"] file_name, root_pid = item["target"] From f95563aaca8c3c720e1cb13c77f950291de4de3b Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:26:56 +0800 Subject: [PATCH 05/15] feat(text): publish independent font coverage and payloads --- docs/pipeline.md | 29 ++++++++- pyproject.toml | 1 + src/cli/legacy.py | 11 ++++ src/ui/text_fonts.py | 75 ++++++++++++++++++++++ tools/deployment/prepare_fonts.py | 100 ++++++++++++++++++++++++++++++ tools/deployment/upload_oss.py | 25 +++++++- 6 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 src/ui/text_fonts.py create mode 100644 tools/deployment/prepare_fonts.py diff --git a/docs/pipeline.md b/docs/pipeline.md index 184810b..9204863 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -121,13 +121,36 @@ only the current pointer. Replacing an existing pointer also requires `--expect-previous-sha256` for the exact previous bytes. Dry-run remains the default; `--execute --exclusive-writer` explicitly enables writes. Retain old objects and roots for rollback; ordinary resource deployment excludes this tree. -This preparation path handles text-only publications; additional font delivery -requires a separately reviewed font publication rather than silently omitting -font references. +This preparation path handles text-only publications. Fonts use the independent +catalogue described below rather than being copied into translation revisions. These exports prepare inputs for the editorial and playback systems. They do not publish translations or grant approval to drafts. +## Independent font catalogue + +An operator prepares the approved open font once. Translators continue to enter +text; they do not upload font files. Coverage is read from each supplied font's +character map, and the base font is identified by the exact UI font hash: + +```sh +pip install 'moly-root[fonts]' +moly export text-fonts --base-font UI.subset.ttf --font OPEN-FONT.ttf \ + --license OFL-1.1.txt --out-dir font-publication/fonts +python tools/deployment/prepare_fonts.py --publication font-publication/fonts \ + --output font-staging --object-prefix sekai-extra-assets/ --url-prefix /moly/ +``` + +`fonts/latest.json` and `fonts/revisions/.json` contain the +`moly-text-fonts/1` catalogue, base-font coverage, and immutable font and license +references under `fonts/objects/`. Upload stable objects first, then use the +existing uploader's explicit mutable scope for `fonts/latest.json`, checking +the previous pointer hash when replacing it. No font object changes when a +translator changes text. Retain historical catalogues and referenced objects. +Set NextTrans's `MYSEKAI_FONT_CATALOG_URL` to the public catalogue so its coverage +check and the stage use the same approved fonts. This command only prepares +files and does not deploy or configure the translation service. + ## One store, two kinds of roots ```text diff --git a/pyproject.toml b/pyproject.toml index bc5e959..5ddc337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ ] [project.optional-dependencies] +fonts = ["fonttools>=4.60,<5"] deployment = [ "brotli>=1.2,<2", "oss2>=2.19,<3", diff --git a/src/cli/legacy.py b/src/cli/legacy.py index 1b471f4..71bdfe1 100644 --- a/src/cli/legacy.py +++ b/src/cli/legacy.py @@ -671,6 +671,11 @@ def main(argv=None): fixture_animations.add_argument("--plan", action="store_true") capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") capabilities.add_argument("--asset-root", required=True) + fonts = sub.add_parser("text-fonts", help="prepare an independent display-font catalogue") + fonts.add_argument("--base-font", action="append", required=True) + fonts.add_argument("--font", action="append", required=True) + fonts.add_argument("--license", required=True) + fonts.add_argument("--out-dir", required=True) tutorial = sub.add_parser("tutorial-placements", help="export source NPC tutorial placement poses") tutorial.add_argument("--apk", action="append", required=True) tutorial.add_argument("--client-version", required=True) @@ -1259,6 +1264,12 @@ def main(argv=None): print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) return 0 + if args.cmd == "text-fonts": + from ui.text_fonts import build_font_catalog + result = build_font_catalog(args.base_font, args.font, args.license, args.out_dir) + print(json.dumps({"revision": result['revision'], "fonts": len(result['fonts'])})) + return 0 + if args.cmd == "tutorial-placements": from chara.tutorial import export_placements result = export_placements(args.apk, args.out_dir, client_version=args.client_version, diff --git a/src/ui/text_fonts.py b/src/ui/text_fonts.py new file mode 100644 index 0000000..6cb36ee --- /dev/null +++ b/src/ui/text_fonts.py @@ -0,0 +1,75 @@ +"""Independent, immutable display-font catalogue from operator-supplied fonts.""" +from pathlib import Path + +from core.content import canonical_bytes, content_hash, document_hash + + +def coverage_ranges(points): + result = [] + for point in sorted(set(points)): + if not 0 <= point <= 0x10FFFF or 0xD800 <= point <= 0xDFFF: + continue + if result and result[-1][1] + 1 == point: + result[-1][1] = point + else: + result.append([point, point]) + return result + + +def build_font_catalog(base_fonts, fonts, license_file, output): + from fontTools.ttLib import TTFont + from io import BytesIO + + output = Path(output) + files = {} + + def font(path): + data = Path(path).read_bytes() + if not 0 < len(data) <= 32 * 1024 * 1024: + raise ValueError('text font is empty or exceeds the runtime font limit') + with TTFont(BytesIO(data), lazy=False) as parsed: + if parsed.flavor or 'glyf' not in parsed: + raise ValueError('text fonts must be uncompressed TrueType fonts') + # Glyph zero is the missing-glyph marker, not coverage. + points = [point for point, glyph in parsed.getBestCmap().items() if parsed.getGlyphID(glyph) != 0] + return data, coverage_ranges(points) + + license_data = Path(license_file).read_bytes() + license_data.decode('utf-8', errors='strict') + if not 0 < len(license_data) <= 65536: + raise ValueError('font license is empty or too large') + + def store(data, suffix): + digest = content_hash(data) + path = f'objects/{digest}.{suffix}' + files[path] = data + return {'path': path, 'sha256': digest, 'bytes': len(data)} + + license_ref = store(license_data, 'txt') + base = {} + for path in base_fonts: + data, coverage = font(path) + base[content_hash(data)] = {'coverage': coverage} + supplemental = [] + for path in fonts: + data, coverage = font(path) + supplemental.append({**store(data, 'ttf'), 'coverage': coverage, 'license': license_ref}) + if not 1 <= len(base) <= 16 or not 1 <= len(supplemental) <= 8: + raise ValueError('font catalogue needs 1-16 base fonts and 1-8 supplemental fonts') + if len({row['sha256'] for row in supplemental}) != len(supplemental): + raise ValueError('duplicate supplemental font') + root = {'schema': 'moly-text-fonts/1', 'baseFonts': base, 'fonts': supplemental} + root['revision'] = document_hash(root) + data = canonical_bytes(root) + if len(data) > 2 * 1024 * 1024: + raise ValueError('font catalogue exceeds its document size limit') + files[f"revisions/{root['revision']}.json"] = data + files['latest.json'] = data + for name, content in files.items(): + path = output / name + if path.exists() and path.read_bytes() != content: + raise ValueError('font output differs; use a fresh publication directory') + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_bytes(content) + return root diff --git a/tools/deployment/prepare_fonts.py b/tools/deployment/prepare_fonts.py new file mode 100644 index 0000000..c331672 --- /dev/null +++ b/tools/deployment/prepare_fonts.py @@ -0,0 +1,100 @@ +"""Prepare an independent immutable text-font publication; no cloud writes.""" +import argparse +import hashlib +import json +from pathlib import Path +import re + +HASH = re.compile(r'[a-f0-9]{64}') + + +def prepare(publication, output, *, object_prefix, url_prefix): + publication = Path(publication).resolve() + output = Path(output).resolve() + data = (publication / 'latest.json').read_bytes() + if len(data) > 2 * 1024 * 1024: + raise ValueError('font catalogue exceeds document limit') + root = json.loads(data) + if (root.get('schema') != 'moly-text-fonts/1' or not HASH.fullmatch(root.get('revision', '')) + or not isinstance(root.get('baseFonts'), dict) or not 1 <= len(root['baseFonts']) <= 16 + or not isinstance(root.get('fonts'), list) or not 1 <= len(root['fonts']) <= 8): + raise ValueError('invalid font catalogue') + identity = {k: v for k, v in root.items() if k != 'revision'} + canonical = (json.dumps(identity, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode() + if hashlib.sha256(canonical).hexdigest() != root['revision']: + raise ValueError('font catalogue revision mismatch') + + def coverage(rows): + if not isinstance(rows, list) or not rows or len(rows) > 65536: + raise ValueError('invalid font coverage ranges') + previous = -1 + for row in rows: + if (not isinstance(row, list) or len(row) != 2 or any(type(n) is not int for n in row) + or not previous < row[0] <= row[1] <= 0x10FFFF + or (row[0] <= 0xDFFF and row[1] >= 0xD800)): + raise ValueError('invalid or overlapping font coverage') + previous = row[1] + + for sha, entry in root['baseFonts'].items(): + if not HASH.fullmatch(sha): + raise ValueError('invalid base font identity') + coverage(entry.get('coverage')) + files = {'latest.json': data, f"revisions/{root['revision']}.json": data} + + def read(ref, suffix, maximum): + if (not isinstance(ref, dict) or not HASH.fullmatch(ref.get('sha256', '')) + or ref.get('path') != f"objects/{ref['sha256']}.{suffix}" + or type(ref.get('bytes')) is not int or not 0 < ref['bytes'] <= maximum): + raise ValueError('invalid immutable font object') + path = (publication / ref['path']).resolve() + if not path.is_relative_to(publication): + raise ValueError('font object escapes publication') + raw = path.read_bytes() + if len(raw) != ref['bytes'] or hashlib.sha256(raw).hexdigest() != ref['sha256']: + raise ValueError('font object integrity mismatch') + files[ref['path']] = raw + + for entry in root['fonts']: + coverage(entry.get('coverage')) + read(entry, 'ttf', 32 * 1024 * 1024) + read(entry.get('license'), 'txt', 65536) + if (publication / f"revisions/{root['revision']}.json").read_bytes() != data: + raise ValueError('archived font catalogue differs from latest') + if (not object_prefix.endswith('/') or object_prefix.startswith('/') or not url_prefix.startswith('/') + or not url_prefix.endswith('/') or any(part in {'.', '..'} for value in [object_prefix, url_prefix] for part in value.split('/')) + or any(char in object_prefix + url_prefix for char in '\\%?#:')): + raise ValueError('publication prefixes must be directories') + rows = [] + for name, raw in sorted(files.items()): + logical = 'fonts/' + name + target = output / 'staging' / logical + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != raw: + raise ValueError('staging bytes differ; choose a fresh directory') + if not target.exists(): + target.write_bytes(raw) + sha = hashlib.sha256(raw).hexdigest() + content_type = 'font/ttf' if name.endswith('.ttf') else 'text/plain; charset=utf-8' if name.endswith('.txt') else 'application/json; charset=utf-8' + rows.append({'path': logical, 'urlPath': url_prefix + logical, 'objectKey': object_prefix + logical, + 'source': str(target), 'sourceRaw': str(target), 'staging': str(target), + 'sha256': sha, 'decodedSha256': sha, 'bytes': len(raw), 'decodedBytes': len(raw), + 'contentEncoding': 'identity', 'contentType': content_type, + 'cacheControl': 'public, max-age=30, must-revalidate' if name == 'latest.json' + else 'public, max-age=31536000, immutable'}) + result = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, + 'urlPrefix': url_prefix, 'snapshots': [], 'fontCatalog': {'path': 'fonts/latest.json', + 'revision': root['revision'], 'sha256': hashlib.sha256(data).hexdigest()}, + 'files': rows, 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows)}} + (output / 'upload-manifest.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--publication', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--object-prefix', default='sekai-extra-assets/') + parser.add_argument('--url-prefix', default='/moly/') + args = parser.parse_args() + result = prepare(args.publication, args.output, object_prefix=args.object_prefix, url_prefix=args.url_prefix) + print(json.dumps({'fontCatalog': result['fontCatalog'], 'totals': result['totals']})) diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index f73f938..f0e6da3 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -24,7 +24,7 @@ REGION = 'cn-shanghai' ENDPOINT = 'https://oss-cn-shanghai.aliyuncs.com' PREFIX = 'moly/alpha/' -MUTABLE = {'manifest.json', 'cache-worker.mjs', 'game/manifest.json'} +MUTABLE = {'manifest.json', 'cache-worker.mjs', 'game/manifest.json', 'fonts/latest.json'} TRANSLATION_ROOT = re.compile(r'translations/(jp|cn|tw|en|kr)/(zh-CN|zh-TW|en-US|ja-JP|ko-KR)/latest\.json') @@ -177,6 +177,13 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) raise SafeFailure('unsupported_delivery_mode') snapshots = set(doc.get('snapshots', [])) translation = doc.get('translation') + font_catalog = doc.get('fontCatalog') + if font_catalog is not None: + if (not isinstance(font_catalog, dict) or snapshots or translation is not None + or font_catalog.get('path') != 'fonts/latest.json' + or not SHA256.fullmatch(font_catalog.get('revision', '')) + or not SHA256.fullmatch(font_catalog.get('sha256', ''))): + raise SafeFailure('invalid_font_publication') if translation is not None: if (not isinstance(translation, dict) or snapshots or not TRANSLATION_ROOT.fullmatch(translation.get('path', '')) @@ -186,7 +193,7 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) translation_base = translation['path'].removesuffix('latest.json') else: translation_base = None - if (not snapshots and translation is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): + if (not snapshots and translation is None and font_catalog is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): raise SafeFailure('invalid_snapshot_identities') if args.scope == 'release' and (args.release_id != doc.get('release') or not IDENTIFIER.fullmatch(args.release_id or '')): raise SafeFailure('explicit_release_id_must_match_manifest') @@ -204,6 +211,8 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected, seen = [], set() for row in doc.get('files', []): logical = row.get('path', '') + if font_catalog is not None and not re.fullmatch(r'fonts/(?:latest\.json|revisions/[a-f0-9]{64}\.json|objects/[a-f0-9]{64}\.(?:ttf|txt))', logical): + raise SafeFailure('font_manifest_contains_unrelated_object') if translation_base and (not logical.startswith(translation_base) or not re.fullmatch( r'(?:latest\.json|(?:objects|revisions)/[a-f0-9]{64}\.json)', logical[len(translation_base):])): raise SafeFailure('translation_manifest_contains_unrelated_object') @@ -241,8 +250,9 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) # The shared content-addressed stores are immutable like snapshot payloads. shared_store = len(parts) >= 2 and parts[0] in {'asset-store', 'catalog-store'} translation_object = translation_base and logical.startswith(translation_base) and logical != translation['path'] + font_object = font_catalog is not None and logical != font_catalog['path'] choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog', 'text'} and len(parts) >= 4) - or shared_store or translation_object)) + or shared_store or translation_object or font_object)) or (args.scope == 'descriptors' and is_snapshot and len(parts) == 3 and parts[2] in {'snapshot.json', 'provenance.json'}) or (args.scope == 'release' and len(parts) >= 3 and parts[:2] == ('releases', args.release_id)) or (args.scope == 'mutable' and logical in args.mutable_path)) @@ -253,6 +263,15 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected = selected[:args.limit] if not selected: raise SafeFailure('no_objects_selected') + if font_catalog is not None: + roots = [row for row in doc['files'] if row['path'] == font_catalog['path']] + if len(roots) != 1 or roots[0]['decodedSha256'] != font_catalog['sha256']: + raise SafeFailure('font_pointer_identity_mismatch') + raw = Path(roots[0]['sourceRaw']).read_bytes() + root = json.loads(raw) + if (hashlib.sha256(raw).hexdigest() != font_catalog['sha256'] + or root.get('schema') != 'moly-text-fonts/1' or root.get('revision') != font_catalog['revision']): + raise SafeFailure('local_font_pointer_changed') if translation is not None: roots = [row for row in doc['files'] if row['path'] == translation['path']] if len(roots) != 1 or roots[0]['decodedSha256'] != translation['sha256']: From 51f3a70bfd8cdc6479664f67a515fb0b0dcafec5 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:52:38 +0800 Subject: [PATCH 06/15] fix(fonts): validate complete publication before upload --- tests/test_font_publication.py | 189 ++++++++++++++++++++++++++++++ tools/deployment/prepare_fonts.py | 82 +++++++++---- tools/deployment/upload_oss.py | 78 ++++++++++-- 3 files changed, 318 insertions(+), 31 deletions(-) create mode 100644 tests/test_font_publication.py diff --git a/tests/test_font_publication.py b/tests/test_font_publication.py new file mode 100644 index 0000000..91fc21c --- /dev/null +++ b/tests/test_font_publication.py @@ -0,0 +1,189 @@ +"""Reject damaged or edited font releases before either upload scope writes.""" +import hashlib +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def canonical(value): + return (json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':')) + '\n').encode() + + +def load_tool(name): + path = Path(__file__).parents[1] / 'tools' / 'deployment' / (name + '.py') + spec = importlib.util.spec_from_file_location('font_test_' + name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def publication(tmp_path, monkeypatch): + tool_dir = Path(__file__).parents[1] / 'tools' / 'deployment' + monkeypatch.syspath_prepend(str(tool_dir)) + prepare, uploader = load_tool('prepare_fonts'), load_tool('upload_oss') + root_dir = tmp_path / 'publication' + (root_dir / 'objects').mkdir(parents=True) + + def object_ref(raw, suffix): + sha = hashlib.sha256(raw).hexdigest() + name = f'objects/{sha}.{suffix}' + (root_dir / name).write_bytes(raw) + return {'path': name, 'sha256': sha, 'bytes': len(raw)} + + license_ref = object_ref(b'Synthetic font license\n', 'txt') + font = {**object_ref(b'\x00\x01\x00\x00synthetic font payload', 'ttf'), + 'coverage': [[32, 126], [0x4e02, 0x4e02]], 'license': license_ref} + root = {'schema': 'moly-text-fonts/1', 'baseFonts': {'b' * 64: {'coverage': [[32, 126]]}}, + 'fonts': [font]} + root['revision'] = hashlib.sha256(canonical(root)).hexdigest() + raw = canonical(root) + (root_dir / 'revisions').mkdir() + (root_dir / f"revisions/{root['revision']}.json").write_bytes(raw) + (root_dir / 'latest.json').write_bytes(raw) + output = tmp_path / 'prepared' + doc = prepare.prepare(root_dir, output, object_prefix='font-test/', url_prefix='/moly/') + manifest = output / 'upload-manifest.json' + + def select(scope='stable', limit=None): + manifest.write_bytes(canonical(doc)) + args = SimpleNamespace(manifest=manifest, scope=scope, release_id=None, + mutable_path=['fonts/latest.json'] if scope == 'mutable' else [], + allow_mutable_update=scope == 'mutable', expect_previous_sha256=None, limit=limit) + return uploader.load_selection(args, object_prefix='font-test/', schemas=('moly-object-upload/1',))[1] + + return SimpleNamespace(doc=doc, root=root, raw=raw, select=select, uploader=uploader, + prepare=prepare, directory=root_dir) + + +def row(publication, suffix): + return next(value for value in publication.doc['files'] if value['path'].endswith(suffix)) + + +def test_complete_publication_has_separate_immutable_and_pointer_scopes(publication): + stable = publication.select() + assert len(stable) == 3 + assert all(value['path'] != 'fonts/latest.json' for value in stable) + assert [value['path'] for value in publication.select('mutable')] == ['fonts/latest.json'] + publication.doc['files'].reverse() + assert publication.select() == stable + + +@pytest.mark.parametrize('suffix', ['.ttf', '.txt', '/latest.json', '/revisions/']) +@pytest.mark.parametrize('scope', ['stable', 'mutable']) +def test_missing_catalogue_dependency_is_refused(publication, suffix, scope): + publication.doc['files'] = [value for value in publication.doc['files'] + if not (suffix in value['path'] if suffix == '/revisions/' + else value['path'].endswith(suffix))] + with pytest.raises(publication.uploader.SafeFailure): + publication.select(scope) + + +def test_extra_or_renamed_immutable_font_cannot_be_published(publication): + extra = dict(row(publication, '.ttf')) + extra['path'] = 'fonts/objects/' + '0' * 64 + '.ttf' + extra['objectKey'] = publication.doc['objectPrefix'] + extra['path'] + extra['urlPath'] = publication.doc['urlPrefix'] + extra['path'] + publication.doc['files'].append(extra) + with pytest.raises(publication.uploader.SafeFailure, match='font_publication_closure_mismatch'): + publication.select() + publication.doc['files'].remove(row(publication, '.ttf')) + with pytest.raises(publication.uploader.SafeFailure, match='font_publication_closure_mismatch'): + publication.select() + + +@pytest.mark.parametrize(('field', 'value'), [ + ('sha256', '0' * 64), ('decodedSha256', '0' * 64), ('bytes', 1), ('decodedBytes', 1), + ('contentEncoding', 'gzip'), ('contentType', 'application/octet-stream'), + ('cacheControl', 'public, max-age=0'), ('urlPath', '/other/font.ttf'), +]) +def test_object_metadata_must_match_catalogue(publication, field, value): + row(publication, '.ttf')[field] = value + with pytest.raises(publication.uploader.SafeFailure, match='font_object_metadata_mismatch'): + publication.select() + + +@pytest.mark.parametrize('alias', ['sourceRaw', 'source', 'staging']) +def test_source_alias_cannot_hide_different_upload_bytes(publication, alias): + row(publication, '/latest.json')[alias] = row(publication, '.txt')['staging'] + with pytest.raises(publication.uploader.SafeFailure): + publication.select('mutable') + + +def test_forged_upload_hash_cannot_override_validated_pointer(publication): + latest, license_row = row(publication, '/latest.json'), row(publication, '.txt') + for field in ('source', 'staging', 'sha256', 'bytes'): + latest[field] = license_row[field] + with pytest.raises(publication.uploader.SafeFailure, match='font_object_metadata_mismatch'): + publication.select('mutable') + + +@pytest.mark.parametrize('suffix', ['.ttf', '.txt', '/latest.json', '/revisions/']) +def test_changed_source_bytes_fail_before_scope_or_canary_upload(publication, suffix): + target = (next(value for value in publication.doc['files'] if suffix in value['path']) + if suffix == '/revisions/' else row(publication, suffix)) + Path(target['staging']).write_bytes(b'changed after preparation') + with pytest.raises(publication.uploader.SafeFailure): + publication.select('mutable') + with pytest.raises(publication.uploader.SafeFailure): + publication.select(limit=1) + + +def test_root_cannot_reuse_revision_after_coverage_change(publication): + publication.root['baseFonts']['b' * 64]['coverage'].append([0x4e02, 0x4e02]) + changed = canonical(publication.root) + for entry in publication.doc['files']: + if entry['path'].endswith('.json'): + Path(entry['staging']).write_bytes(changed) + entry['sha256'] = entry['decodedSha256'] = hashlib.sha256(changed).hexdigest() + entry['bytes'] = entry['decodedBytes'] = len(changed) + publication.doc['fontCatalog']['sha256'] = hashlib.sha256(changed).hexdigest() + with pytest.raises(publication.uploader.SafeFailure, match='invalid_local_font_catalogue'): + publication.select('mutable') + + +def test_pointer_row_cannot_publish_a_different_canonical_root(publication): + # Keep the original sourceRaw and catalogue metadata, but point the upload + # stream at a second, internally valid root. Both files parse; only one is + # the publication selected for this run. + publication.root.pop('revision') + publication.root['baseFonts']['b' * 64]['coverage'] = [[32, 125]] + publication.root['revision'] = hashlib.sha256(canonical(publication.root)).hexdigest() + alternate = publication.directory / 'alternate.json' + alternate.write_bytes(canonical(publication.root)) + latest = row(publication, '/latest.json') + latest['staging'] = str(alternate) + latest['sha256'] = hashlib.sha256(alternate.read_bytes()).hexdigest() + latest['bytes'] = alternate.stat().st_size + with pytest.raises(publication.uploader.SafeFailure, match='font_object_metadata_mismatch'): + publication.select('mutable') + + +@pytest.mark.parametrize('encoding', ['utf-16', 'utf-32']) +def test_non_utf8_root_cannot_publish_to_utf8_consumers(publication, encoding): + changed = publication.raw.decode('utf-8').encode(encoding) + for entry in publication.doc['files']: + if entry['path'].endswith('.json'): + Path(entry['staging']).write_bytes(changed) + entry['sha256'] = entry['decodedSha256'] = hashlib.sha256(changed).hexdigest() + entry['bytes'] = entry['decodedBytes'] = len(changed) + publication.doc['fontCatalog']['sha256'] = hashlib.sha256(changed).hexdigest() + with pytest.raises(publication.uploader.SafeFailure, match='invalid_local_font_catalogue'): + publication.select('mutable') + + +@pytest.mark.parametrize('mutate', [ + lambda root: root['fonts'].append(dict(root['fonts'][0])), + lambda root: root['fonts'][0].update(coverage=[[100, 110], [99, 99]]), + lambda root: root['fonts'][0].update(coverage=[[0xd800, 0xd800]]), + lambda root: root['fonts'][0].update(path='objects/' + '0' * 64 + '.ttf'), +]) +def test_preparation_and_uploader_share_invalid_catalogue_gate(publication, mutate): + mutate(publication.root) + publication.root.pop('revision') + publication.root['revision'] = hashlib.sha256(canonical(publication.root)).hexdigest() + with pytest.raises(ValueError): + publication.prepare.parse_catalog(canonical(publication.root)) diff --git a/tools/deployment/prepare_fonts.py b/tools/deployment/prepare_fonts.py index c331672..84f12c2 100644 --- a/tools/deployment/prepare_fonts.py +++ b/tools/deployment/prepare_fonts.py @@ -8,14 +8,33 @@ HASH = re.compile(r'[a-f0-9]{64}') -def prepare(publication, output, *, object_prefix, url_prefix): - publication = Path(publication).resolve() - output = Path(output).resolve() - data = (publication / 'latest.json').read_bytes() +def content_type(path): + return ('font/ttf' if path.endswith('.ttf') else 'text/plain; charset=utf-8' + if path.endswith('.txt') else 'application/json; charset=utf-8') + + +def cache_control(path): + return ('public, max-age=30, must-revalidate' if path == 'latest.json' + else 'public, max-age=31536000, immutable') + + +def parse_catalog(data): + """Validate the root and return its complete, content-addressed file closure. + + Used both during preparation and immediately before an upload selection. + Font bytes remain with the caller so upload validation can stream them. + """ if len(data) > 2 * 1024 * 1024: raise ValueError('font catalogue exceeds document limit') - root = json.loads(data) - if (root.get('schema') != 'moly-text-fonts/1' or not HASH.fullmatch(root.get('revision', '')) + # Browser/Go consumers decode UTF-8, whereas json.loads(bytes) also accepts + # UTF-16/32. Reject encodings that would publish successfully but not load. + root = json.loads(data.decode('utf-8', errors='strict')) + + def is_hash(value): + return isinstance(value, str) and HASH.fullmatch(value) + + if (not isinstance(root, dict) or root.get('schema') != 'moly-text-fonts/1' + or not is_hash(root.get('revision')) or not isinstance(root.get('baseFonts'), dict) or not 1 <= len(root['baseFonts']) <= 16 or not isinstance(root.get('fonts'), list) or not 1 <= len(root['fonts']) <= 8): raise ValueError('invalid font catalogue') @@ -36,30 +55,49 @@ def coverage(rows): previous = row[1] for sha, entry in root['baseFonts'].items(): - if not HASH.fullmatch(sha): + if not is_hash(sha) or not isinstance(entry, dict): raise ValueError('invalid base font identity') coverage(entry.get('coverage')) - files = {'latest.json': data, f"revisions/{root['revision']}.json": data} + root_ref = {'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data)} + files = {'latest.json': root_ref, f"revisions/{root['revision']}.json": root_ref} - def read(ref, suffix, maximum): - if (not isinstance(ref, dict) or not HASH.fullmatch(ref.get('sha256', '')) + def reference(ref, suffix, maximum): + if (not isinstance(ref, dict) or not is_hash(ref.get('sha256')) or ref.get('path') != f"objects/{ref['sha256']}.{suffix}" or type(ref.get('bytes')) is not int or not 0 < ref['bytes'] <= maximum): raise ValueError('invalid immutable font object') - path = (publication / ref['path']).resolve() + expected = {'sha256': ref['sha256'], 'bytes': ref['bytes']} + if ref['path'] in files and files[ref['path']] != expected: + raise ValueError('conflicting font object reference') + files[ref['path']] = expected + + seen = set() + for entry in root['fonts']: + if not isinstance(entry, dict): + raise ValueError('invalid supplemental font') + coverage(entry.get('coverage')) + reference(entry, 'ttf', 32 * 1024 * 1024) + reference(entry.get('license'), 'txt', 65536) + if entry['sha256'] in seen: + raise ValueError('duplicate supplemental font') + seen.add(entry['sha256']) + return root, files + + +def prepare(publication, output, *, object_prefix, url_prefix): + publication = Path(publication).resolve() + output = Path(output).resolve() + data = (publication / 'latest.json').read_bytes() + root, references = parse_catalog(data) + files = {} + for name, ref in references.items(): + path = (publication / name).resolve() if not path.is_relative_to(publication): raise ValueError('font object escapes publication') raw = path.read_bytes() if len(raw) != ref['bytes'] or hashlib.sha256(raw).hexdigest() != ref['sha256']: raise ValueError('font object integrity mismatch') - files[ref['path']] = raw - - for entry in root['fonts']: - coverage(entry.get('coverage')) - read(entry, 'ttf', 32 * 1024 * 1024) - read(entry.get('license'), 'txt', 65536) - if (publication / f"revisions/{root['revision']}.json").read_bytes() != data: - raise ValueError('archived font catalogue differs from latest') + files[name] = raw if (not object_prefix.endswith('/') or object_prefix.startswith('/') or not url_prefix.startswith('/') or not url_prefix.endswith('/') or any(part in {'.', '..'} for value in [object_prefix, url_prefix] for part in value.split('/')) or any(char in object_prefix + url_prefix for char in '\\%?#:')): @@ -74,13 +112,11 @@ def read(ref, suffix, maximum): if not target.exists(): target.write_bytes(raw) sha = hashlib.sha256(raw).hexdigest() - content_type = 'font/ttf' if name.endswith('.ttf') else 'text/plain; charset=utf-8' if name.endswith('.txt') else 'application/json; charset=utf-8' rows.append({'path': logical, 'urlPath': url_prefix + logical, 'objectKey': object_prefix + logical, 'source': str(target), 'sourceRaw': str(target), 'staging': str(target), 'sha256': sha, 'decodedSha256': sha, 'bytes': len(raw), 'decodedBytes': len(raw), - 'contentEncoding': 'identity', 'contentType': content_type, - 'cacheControl': 'public, max-age=30, must-revalidate' if name == 'latest.json' - else 'public, max-age=31536000, immutable'}) + 'contentEncoding': 'identity', 'contentType': content_type(name), + 'cacheControl': cache_control(name)}) result = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, 'urlPrefix': url_prefix, 'snapshots': [], 'fontCatalog': {'path': 'fonts/latest.json', 'revision': root['revision'], 'sha256': hashlib.sha256(data).hexdigest()}, diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index f0e6da3..2723ec2 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -166,6 +166,75 @@ def parse_args(): return p.parse_args() +def validate_font_publication(doc): + # Preparation is not a trust boundary: manifests and staging paths can + # change before execution. Reuse its catalogue contract, then bind every + # declared input and the actual upload stream to the full root closure. + from prepare_fonts import parse_catalog, content_type, cache_control + + catalog = doc['fontCatalog'] + rows = {row['path']: row for row in doc['files']} + root_row = rows.get(catalog['path']) + if root_row is None: + raise SafeFailure('font_pointer_identity_mismatch') + try: + with Path(root_row['sourceRaw']).open('rb') as stream: + raw = stream.read(2 * 1024 * 1024 + 1) + root, references = parse_catalog(raw) + except (OSError, ValueError, KeyError, TypeError) as error: + raise SafeFailure('invalid_local_font_catalogue', errorType=type(error).__name__) from None + if (root['revision'] != catalog['revision'] + or references['latest.json']['sha256'] != catalog['sha256']): + raise SafeFailure('font_pointer_identity_mismatch') + expected_paths = {'fonts/' + name for name in references} + if rows.keys() != expected_paths: + raise SafeFailure('font_publication_closure_mismatch', + missing=sorted(expected_paths - rows.keys()), + extra=sorted(rows.keys() - expected_paths)) + url_prefix = doc.get('urlPrefix') + if (not isinstance(url_prefix, str) or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for part in url_prefix.split('/')) + or any(char in url_prefix for char in '\\%?#:') + or any(ord(char) < 32 or ord(char) == 127 for char in url_prefix)): + raise SafeFailure('invalid_font_url_prefix') + verified = set() + for name, expected in references.items(): + logical = 'fonts/' + name + row = rows[logical] + if (row['sha256'] != expected['sha256'] or row['decodedSha256'] != expected['sha256'] + or row['bytes'] != expected['bytes'] or row['decodedBytes'] != expected['bytes'] + or row['contentEncoding'] != 'identity' or row['contentType'] != content_type(name) + or row['cacheControl'] != cache_control(name) + or row.get('urlPath') != url_prefix + logical): + raise SafeFailure('font_object_metadata_mismatch', path=logical) + # All aliases are checked; sourceRaw must not validate one document + # while staging/source supplies a different body to put_one(). + aliases = [row.get('sourceRaw'), row.get('source')] + if 'staging' in row: + aliases.append(row['staging']) + for alias in aliases: + if not isinstance(alias, str) or not alias: + raise SafeFailure('invalid_font_object_source', path=logical) + try: + path = Path(alias).resolve(strict=True) + identity = (path, expected['sha256'], expected['bytes']) + if identity in verified: + continue + digest, size = hashlib.sha256(), 0 + with path.open('rb') as stream: + while block := stream.read(1024 * 1024): + size += len(block) + if size > expected['bytes']: + raise SafeFailure('local_font_object_changed', path=logical) + digest.update(block) + if size != expected['bytes'] or digest.hexdigest() != expected['sha256']: + raise SafeFailure('local_font_object_changed', path=logical) + verified.add(identity) + except OSError as error: + raise SafeFailure('cannot_read_font_object', path=logical, + errorType=type(error).__name__) from None + + def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',)): try: doc = json.loads(args.manifest.read_bytes()) @@ -264,14 +333,7 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) if not selected: raise SafeFailure('no_objects_selected') if font_catalog is not None: - roots = [row for row in doc['files'] if row['path'] == font_catalog['path']] - if len(roots) != 1 or roots[0]['decodedSha256'] != font_catalog['sha256']: - raise SafeFailure('font_pointer_identity_mismatch') - raw = Path(roots[0]['sourceRaw']).read_bytes() - root = json.loads(raw) - if (hashlib.sha256(raw).hexdigest() != font_catalog['sha256'] - or root.get('schema') != 'moly-text-fonts/1' or root.get('revision') != font_catalog['revision']): - raise SafeFailure('local_font_pointer_changed') + validate_font_publication(doc) if translation is not None: roots = [row for row in doc['files'] if row['path'] == translation['path']] if len(roots) != 1 or roots[0]['decodedSha256'] != translation['sha256']: From e634cbdc34d93d91b5fba73cc5edaeba46e11a60 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:28:56 +0800 Subject: [PATCH 07/15] feat(character): retain per-actor animation source identities --- src/chara/characters.py | 65 +++++++++++++++++++++++ src/chara/registry.py | 12 ++++- tests/test_character_animation_map.py | 76 +++++++++++++++++++++++++++ tests/test_registry.py | 20 +++++++ tools/migrate_canonical_metadata.py | 7 +-- 5 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 tests/test_character_animation_map.py diff --git a/src/chara/characters.py b/src/chara/characters.py index 0e59752..e3175f7 100644 --- a/src/chara/characters.py +++ b/src/chara/characters.py @@ -88,6 +88,69 @@ def one(self, type_name): return objs[0] +def read_animation_controller(assets): + """Exact clip membership of the character's controller-bearing Animator. + + Auxiliary Animators without controllers are present in character packages. + A missing controller stays null; multiple concrete owners are ambiguous. + Clip references retain their serialized order and source identities, so a + consumer can join names and authored timing from the shared motion index + without treating its union of clips as this particular actor's repertoire. + """ + def pointer(owner, value, *, allow_null=False): + if not isinstance(value, dict): + raise ValueError('character animation reference is not a PPtr') + file_id, path_id = value.get('m_FileID'), value.get('m_PathID') + if (type(file_id) is not int or file_id < 0 or type(path_id) is not int + or not -(1 << 63) <= path_id < (1 << 63)): + raise ValueError('character animation reference has invalid identifiers') + if path_id == 0: + if allow_null: + return None + raise ValueError('character controller has a null animation clip') + if file_id == 0: + file = owner.assets_file.name + elif file_id <= len(owner.assets_file.externals): + file = owner.assets_file.externals[file_id - 1].path.replace('\\', '/').rsplit('/', 1)[-1] + else: + raise ValueError('character animation reference has no external file') + if not isinstance(file, str) or not file: + raise ValueError('character animation reference has no source file') + return {'file': file, 'pathId': str(path_id)} + + candidates = [] + for obj in assets.objects: + if obj.type.name != 'Animator': + continue + ref = obj.read_typetree().get('m_Controller') + identity = pointer(obj, ref, allow_null=True) + if identity is not None: + candidates.append((obj, ref, identity)) + if not candidates: + return None + if len(candidates) != 1: + raise ValueError('character needs one controller-bearing Animator') + animator, ref, identity = candidates[0] + controller = assets.deref(animator, ref) + if controller is None or controller.type.name != 'AnimatorController': + raise ValueError('character Animator requires its actual AnimatorController') + source = {'file': controller.assets_file.name, 'pathId': str(controller.path_id)} + if source != identity: + raise ValueError('character Animator controller identity mismatch') + references = controller.read_typetree().get('m_AnimationClips') + if not isinstance(references, list): + raise ValueError('character controller has no animation clip table') + clips = [pointer(controller, value) for value in references] + if len({(value['file'], value['pathId']) for value in clips}) != len(clips): + raise ValueError('character controller duplicates an animation clip') + return { + 'schema': 'moly-character-animation-map/1', + 'animator': {'file': animator.assets_file.name, 'pathId': str(animator.path_id)}, + 'controller': source, + 'clips': clips, + } + + def read_scene(assets): """The instantiable prefab transform tree. @@ -522,6 +585,7 @@ def extract_character(bundle, out_dir, name, sampled=None, aux=(), verification numbers (all verifications also raise on hard violations). """ assets = CharacterAssets(bundle, aux) + animation_controller = read_animation_controller(assets) av_tt = assets.one("Avatar").read_typetree() rig = Rig(rig_doc(av_tt)) nodes, tr_index = read_scene(assets) @@ -820,6 +884,7 @@ def wm(i): "cloth": cloth_rig, "anchors": read_anchors(assets, tr_index, nodes), "headIk": head_ik_axes(rig.doc, nodes), + "animationController": animation_controller, "rendererBounds": [{**renderer['bounds'], 'name': renderer['rendererName']} for renderer in renderers], "textures": [tex_paths[n] for n in sorted(pngs)], # materials[i].name's contract is "the glTF material's own name", diff --git a/src/chara/registry.py b/src/chara/registry.py index d8a6ed5..33b9a01 100644 --- a/src/chara/registry.py +++ b/src/chara/registry.py @@ -40,6 +40,10 @@ "gameCharacterId: firstName + givenName, skipping absent parts; " "null when the character row is absent — the caller decides how to " "label a nameless unit, this module never invents one"), + "gameCharacter": ("identity.gameCharacterId joined to the gameCharacters row; " + "id and verbatim gender are kept together. Null means the " + "source row is absent; gender is not inferred from unit, " + "motion family, model name, or available animation files"), "storedVsRuntime": ("the runtime divides pauseMilliSeconds, " "changeMotionMilliSeconds, walkSpeed and runSpeed by 1000; " "both forms are given and the runtime form is the one to use"), @@ -100,11 +104,14 @@ def build_registry(master, unit_ids, motion_library_index=None, client_configs=_ ("soloAction", solo.get(unit))) if row is None] if gaps: missing[str(unit)] = gaps - name = _display_name(game_characters.get(ident_row["gameCharacterId"]) - if (game_characters and ident_row) else None) + character_id = ident_row.get("gameCharacterId") if ident_row else None + character_row = game_characters.get(character_id) if game_characters is not None else None + name = _display_name(character_row) entry = { "unitId": unit, "name": name, + "gameCharacter": ({"id": character_id, "gender": character_row.get("gender")} + if character_row is not None else None), "identity": ({f: ident_row.get(f) for f in IDENTITY_FIELDS} if ident_row else None), "locomotion": None, @@ -160,6 +167,7 @@ def build_registry(master, unit_ids, motion_library_index=None, client_configs=_ "requested": len(set(unit_ids)), "withIdentity": sum(1 for c in characters.values() if c["identity"]), "withName": sum(1 for c in characters.values() if c["name"]), + "withGameCharacter": sum(1 for c in characters.values() if c["gameCharacter"] is not None), "withLocomotion": sum(1 for c in characters.values() if c["locomotion"]), "withSoloAction": sum(1 for c in characters.values() if c["soloAction"]), "missing": missing, diff --git a/tests/test_character_animation_map.py b/tests/test_character_animation_map.py new file mode 100644 index 0000000..bfd2929 --- /dev/null +++ b/tests/test_character_animation_map.py @@ -0,0 +1,76 @@ +"""Controller identity must not widen a character's animation repertoire.""" +from types import SimpleNamespace + +import pytest + +from chara.characters import read_animation_controller + + +def ref(file_id, path_id): + return {"m_FileID": file_id, "m_PathID": path_id} + + +class Object: + def __init__(self, kind, path_id, tree, *, file="CAB-character", externals=()): + self.type = SimpleNamespace(name=kind) + self.path_id = path_id + self.assets_file = SimpleNamespace(name=file, externals=[SimpleNamespace(path=p) for p in externals]) + self.tree = tree + + def read_typetree(self): + return self.tree + + +def assets(clips, *, auxiliary=True): + controller = Object("AnimatorController", -123, {"m_AnimationClips": clips}, + externals=["archive:/CAB-motion/CAB-motion", "archive:\\CAB-other\\CAB-other"]) + animator = Object("Animator", 10, {"m_Controller": ref(0, -123)}) + objects = [animator, controller] + if auxiliary: + objects.insert(0, Object("Animator", 11, {"m_Controller": ref(0, 0)})) + return SimpleNamespace(objects=objects, deref=lambda owner, pointer: controller) + + +def test_actor_controller_keeps_source_membership_order_and_signed_ids(): + source = assets([ref(2, -(1 << 62)), ref(0, 41), ref(1, 41)]) + # A separately listed, unrelated controller must never supply the clips. + source.objects.append(Object("AnimatorController", 99, {"m_AnimationClips": [ref(0, 42)]})) + result = read_animation_controller(source) + assert result["animator"] == {"file": "CAB-character", "pathId": "10"} + assert result["controller"] == {"file": "CAB-character", "pathId": "-123"} + assert result["clips"] == [ + {"file": "CAB-other", "pathId": str(-(1 << 62))}, + {"file": "CAB-character", "pathId": "41"}, + {"file": "CAB-motion", "pathId": "41"}, + ] + # Same numeric path ID in another serialized file is a different clip. + index = {("CAB-character", "41"): "female_S", ("CAB-motion", "41"): "male_S", + ("CAB-other", str(-(1 << 62))): "idle_L"} + assert [index[(r["file"], r["pathId"])] for r in result["clips"]] == ["idle_L", "female_S", "male_S"] + + +def test_missing_controller_stays_null_but_empty_real_controller_stays_present(): + assert read_animation_controller(SimpleNamespace(objects=[])) is None + null = Object("Animator", 1, {"m_Controller": ref(0, 0)}) + assert read_animation_controller(SimpleNamespace(objects=[null])) is None + assert read_animation_controller(assets([]))["clips"] == [] + + +def test_ambiguous_animator_or_changed_controller_identity_is_refused(): + source = assets([]) + source.objects.append(Object("Animator", 12, {"m_Controller": ref(0, -123)})) + with pytest.raises(ValueError, match="one controller-bearing Animator"): + read_animation_controller(source) + source.objects.pop() + source.objects[-1].path_id = -124 + with pytest.raises(ValueError, match="identity mismatch"): + read_animation_controller(source) + + +@pytest.mark.parametrize("clips", [ + [ref(0, 0)], [ref(3, 1)], [ref(-1, 1)], [ref(True, 1)], [ref(1, True)], + [ref(0, 1 << 63)], [ref(1, 4), ref(1, 4)], [None], +]) +def test_invalid_or_duplicate_clip_references_do_not_produce_partial_maps(clips): + with pytest.raises(ValueError): + read_animation_controller(assets(clips)) diff --git a/tests/test_registry.py b/tests/test_registry.py index f277425..8a3741d 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -116,12 +116,32 @@ def test_a_missing_source_row_is_reported_not_defaulted(): def test_a_character_absent_from_every_table_still_appears(): doc = build_registry(_master(), [12, 21]) assert doc["characters"]["21"] == {"unitId": 21, "name": None, + "gameCharacter": None, "identity": None, "locomotion": None, "soloAction": None} assert doc["summary"]["missing"]["21"] == ["identity", "locomotion", "soloAction"] assert doc["summary"]["requested"] == 2 +def test_gender_joins_character_identity_and_preserves_the_source_value(): + master = _master(identity={12: {"gameCharacterId": 7}, 13: {"gameCharacterId": 7}}, + characters={7: {"id": 7, "gender": "MaLe"}, + 12: {"id": 12, "gender": "female"}}) + doc = build_registry(master, [12, 13]) + for unit in ("12", "13"): + assert doc["characters"][unit]["gameCharacter"] == {"id": 7, "gender": "MaLe"} + assert doc["characters"][unit]["identity"]["gameCharacterId"] == 7 + assert doc["summary"]["withGameCharacter"] == 2 + + +def test_missing_character_or_gender_is_not_inferred_from_motion_names(): + absent = build_registry(_master(), [12])["characters"]["12"] + assert absent["identity"]["gameCharacterId"] == 12 + assert absent["gameCharacter"] is None + no_gender = build_registry(_master(characters={12: {"id": 12}}), [12]) + assert no_gender["characters"]["12"]["gameCharacter"] == {"id": 12, "gender": None} + + def test_motion_names_are_checked_against_the_library_when_given(): index = {"clips": {"mov_idle": {}, "mov_walk": {}}} # run 缺 doc = build_registry(_master(), [12], motion_library_index=index) diff --git a/tools/migrate_canonical_metadata.py b/tools/migrate_canonical_metadata.py index a73ab20..ef68147 100644 --- a/tools/migrate_canonical_metadata.py +++ b/tools/migrate_canonical_metadata.py @@ -25,9 +25,10 @@ # They keep the guard usable in source archives and installed source copies. # Update them only after reviewing the coordinate-writing code. REVIEWED_PRODUCERS = { - # Adds sidecar look-at axes and renderer bounds using the existing X - # reflection. Mesh vertices, animation, skinning and GLB writes are unchanged. - 'src/chara/characters.py': 'a7f8e51574dac542f0bc4ea07077736f929eb08634ef385149bcd4269323461d', + # Adds source controller/clip identities to the rig only. Removing that + # read-only helper, its call and the one sidecar field reproduces the prior + # reviewed AST; mesh, animation, skinning and GLB writes are unchanged. + 'src/chara/characters.py': '435068f0dd34ff6cb3c329196e0a4165ff78fa409ee93469dec881aff0d7ad70', 'src/chara/motion_library.py': '3ca5702e6ff3d02ee065c32dc5a2e1fe24716999976c135ffa77e52a3b26465c', 'src/chara/player_avatar.py': 'f10f702f6fc3b58f23c1f1a5a388bbe94789e27fec7d93d2b8316abf13a012e1', 'src/chara/avatar_parts.py': '74a42b5519bae8b645850e0281420c707ebac33fa08df3f5bd6326a4dcf89fc3', From f699eeedf6d747e79e9c1e8cc53d237f042e1e65 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:15:57 +0800 Subject: [PATCH 08/15] feat(export): retain director transforms and photo presentation --- src/perf/directors.py | 5 +- src/ui/jp_decoders.py | 5 ++ src/ui/photo_layout.py | 33 ++++++++++++ src/ui/photo_presentation.py | 98 ++++++++++++++++++++++++++++++++++++ src/ui/prefab_layout.py | 12 ++++- src/ui/talk.py | 2 +- 6 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 src/ui/photo_presentation.py diff --git a/src/perf/directors.py b/src/perf/directors.py index 66102a8..5e046eb 100644 --- a/src/perf/directors.py +++ b/src/perf/directors.py @@ -95,12 +95,13 @@ def _signal_receiver(store, record, component_id, owner, hierarchy): "hierarchy": hierarchy, "reactions": rows} -def _director(store, record, path_id, owner, hierarchy): +def _director(store, record, path_id, owner, hierarchy, transforms): tree = record.tree(path_id) return { "asset": identity(record, path_id), "gameObject": identity(*owner), "hierarchy": hierarchy, + "transforms": transforms, "timeline": reference(store, record, tree.get("m_PlayableAsset")), "enabled": tree.get("m_Enabled"), "updateMode": tree.get("m_DirectorUpdateMode"), @@ -214,7 +215,7 @@ def read_prefab_directors(store, package): component_kind = component_file.kinds[component_id] if component_kind == "PlayableDirector": prefab["directors"].append(_director( - store, component_file, component_id, owner, hierarchy)) + store, component_file, component_id, owner, hierarchy, own)) elif (component_kind == "MonoBehaviour" and component_file.script_of(component_id) in TIMELINE_VIEW_SCRIPTS): prefab["fixtureTimelineViews"].append(_fixture_timeline_view( diff --git a/src/ui/jp_decoders.py b/src/ui/jp_decoders.py index 780b6f6..054109f 100644 --- a/src/ui/jp_decoders.py +++ b/src/ui/jp_decoders.py @@ -562,6 +562,11 @@ def decode_list_view_700(r: talk.Reader) -> dict: # (class name, serialized type hash hex) -> the layout the bytes carry. TYPE_LAYOUTS = { + ("Sekai.UIPartsPhotoFrameThumbnail", "701a1b9241173c653af93358697fbf4a"): "7.0.0", + ("Sekai.Mysekai.MysekaiSelectFrameView", "56a5828cb66f421adde139868f381dc6"): "7.0.0", + ("Sekai.UIPartsPhotoFramePreview", "8d1a4f81285d407da7fc2608b914a06e"): "7.0.0", + ("Sekai.Mysekai.MysekaiPhotoSaveAnimation", "a7952c6374ff1805f356704076fb3a8a"): "7.0.0", + ("Sekai.AnimationClipEvent", "431f36454113f7d83bc74347ac5d008f"): "7.0.0", ("Sekai.Mysekai.MysekaiPhotoShotView", "9c249c4e2d9276be8c161151ab811ebf"): "7.0.0", ("Sekai.Mysekai.MysekaiPhotoShotAnimation", "f1b1ff8667c818bf70d56dd9b284b432"): "7.0.0", ("Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter", "a4805d90303631a00c5a8e37e1ff5a00"): "7.0.0", diff --git a/src/ui/photo_layout.py b/src/ui/photo_layout.py index 65cc2a7..8eaf8a7 100644 --- a/src/ui/photo_layout.py +++ b/src/ui/photo_layout.py @@ -10,6 +10,7 @@ "btn_photoshot_gr_h180", "icon_callout_wh", "icon_camera_wh_h80", "btn_circle120_lgr", "icon_cameraEffect_h56", "icon_tripod_wh_h44", "icon_photoalbum_wh_h42", "txt_photoshot_blur", "txt_photoshot_filmCount", + "tex_mysekai_photo_shot_camera", "tex_mysekai_photo_shot_parts", "img_film_default", ) @@ -89,6 +90,7 @@ def package_effects(bundle, bundle_root, out): for role, prefab in ( ('call-button', 'fx_mysekai_photo_shot_call_character_pt'), ('call-wave', 'fx_mysekai_photo_shot_call_character_wave'), + ('shot', 'fx_mysekai_photo_shot_pt'), ): targets = [(record, path_id) for name, record, path_id in package.contents if name == prefab + '.prefab'] @@ -158,8 +160,39 @@ def decode_presenter(reader: talk.Reader): return editor_layout.refs(reader, ("_photoShotView", "_selectFrameView", "_photoSaveAnimation")) +def decode_select_frame(reader: talk.Reader): + return references(reader, ( + "_canvasGroup", "_photoShotAnimation", "_scroller", + "_selectedPhotoFrameNameText", "_savePhotoButton", "_photoShotSkipButton", + )) + + +def decode_preview(reader: talk.Reader): + return references(reader, ( + "_rootCanvasGroup", "_photoLoader", "_frameLoader", "_frameCanvasGroup", + )) + + +def decode_save_animation(reader: talk.Reader): + return references(reader, ("_canvasGroup", "_photoAlbumPageView")) + + +def decode_thumbnail(reader: talk.Reader): + return references(reader, ("_frameLoader", "_quantityText", "_selectedImage", "_selectedButton")) + + +def decode_clip_event(reader: talk.Reader): + # Callback is a runtime Action, not a serialized UnityEvent. + return {} + + DECODERS = { "Sekai.Mysekai.MysekaiPhotoShotView": decode_view, "Sekai.Mysekai.MysekaiPhotoShotAnimation": decode_animation, "Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter": decode_presenter, + "Sekai.Mysekai.MysekaiSelectFrameView": decode_select_frame, + "Sekai.UIPartsPhotoFramePreview": decode_preview, + "Sekai.Mysekai.MysekaiPhotoSaveAnimation": decode_save_animation, + "Sekai.AnimationClipEvent": decode_clip_event, + "Sekai.UIPartsPhotoFrameThumbnail": decode_thumbnail, } diff --git a/src/ui/photo_presentation.py b/src/ui/photo_presentation.py new file mode 100644 index 0000000..7e1cc4a --- /dev/null +++ b/src/ui/photo_presentation.py @@ -0,0 +1,98 @@ +"""Playable PhotoShot curves and the master's frame image inventory. + +The inventory is not a possession list. Possession is a separate client input. +Curve targets are resolved from the animator's actual bindings to the exported +screen identities; no hierarchy-name lookup is needed by the runtime player. +""" +from pathlib import Path +import hashlib +import json + +import UnityPy +from core.jsonio import write_json + + +def export_presentation(document, master_dir, bundle_root, out): + nodes = document['nodes'] + animation = next(c for n in nodes for c in n['components'] + if c['class'] == 'Sekai.Mysekai.MysekaiPhotoShotAnimation') + animator_id = animation['fields']['_photoShotAnimator'][1] + animator = next(c for n in nodes for c in n['components'] if c['pathId'] == animator_id) + source = animator['animatorController'] + state = next(s for m in source['controller']['stateMachines'] for s in m['states'] + if s['name'] == 'PhotoShot') + if len(state['motions']) != 1 or state['loop'] or state['speed'] != 1: + raise ValueError('PhotoShot requires one non-looping motion at authored speed') + motion = state['motions'][0] + clip = next(c for c in source['clips'] if c['source'] == motion['clip']['source']) + if clip['accounting']['unresolved'] or clip['accounting']['bindingSlots'] != len(clip['curves']): + raise ValueError('PhotoShot has unresolved animation slots') + properties = { + (224, 1460864421): ('position', 0), (224, 538195251): ('position', 1), + (114, 2526845255): ('color', 0), (114, 4215373228): ('color', 1), + (114, 2334886179): ('color', 2), (114, 304273561): ('color', 3), + (225, 1574349066): ('alpha', 0), (1, 2086281974): ('active', 0), + } + targets = {} + curves = [] + for curve in clip['curves']: + if len(curve['targets']) != 1 or curve['kind'] not in ('cubic', 'const', 'linear'): + raise ValueError('PhotoShot curve needs one exact target and supported interpolation') + target, binding = curve['targets'][0], curve['binding'] + game_object_id = int(target['gameObject']['pathId']) + node = next(n for n in nodes if n['gameObjectId'] == game_object_id) + if binding['typeId'] == 4 and binding['attribute'] in (3, 4): + prop, channel = ('scale' if binding['attribute'] == 3 else 'euler'), binding['component'] + else: + prop, channel = properties[(binding['typeId'], binding['attribute'])] + component_id = None + if prop == 'color': + if len(target['components']) != 1: + raise ValueError('PhotoShot color needs its exact Graphic component') + component_id = int(target['components'][0]['pathId']) + key = node['transformId'] + if key not in targets: + graphic = next((c for c in node['components'] if c['pathId'] == component_id), None) + targets[key] = {'transformId': node['transformId'], 'graphicId': component_id, + 'selectorId': next((c['pathId'] for c in node['components'] + if c.get('fields', {}).get('m_Color') is not None), + node['transformId']), + 'position': node['rect']['anchoredPosition'], 'scale': node['rect']['localScale'], + 'color': graphic['fields']['m_Color'] if graphic else [1., 1., 1., 1.]} + if component_id is not None: + graphic = next(c for c in node['components'] if c['pathId'] == component_id) + targets[key]['graphicId'] = component_id + targets[key]['color'] = graphic['fields']['m_Color'] + curves.append({'target': list(targets).index(key), 'property': prop, 'channel': channel, + 'kind': curve['kind'], 'points': curve['points'], 'sourceSlot': curve['slot']}) + frames = [] + output = Path(out) + for row in json.loads((Path(master_dir) / 'mysekaiPhotoDecorations.json').read_text(encoding='utf-8')): + frame = {'id': row['id'], 'name': row['name'], 'assetbundleName': row['assetbundleName']} + for role in ('preview', 'thumbnail'): + package = f"mysekai__photo_frame__{role}__{row['assetbundleName']}" + path = Path(bundle_root) / package + data = path.read_bytes() + env = UnityPy.load(data) + matches = [o for o in env.objects if o.type.name == 'Texture2D' + and o.read_typetree()['m_Name'] == row['assetbundleName']] + if len(matches) != 1: + raise ValueError(f'{package} needs exactly one named frame texture') + texture = matches[0] + relative = f"photo-shot/frames/{row['id']}-{role}.png" + (output / relative).parent.mkdir(parents=True, exist_ok=True) + texture.read().image.save(output / relative) + frame[role] = relative + frame[f'{role}Source'] = {'package': package, 'pathId': texture.path_id, + 'sha256': hashlib.sha256(data).hexdigest()} + frames.append(frame) + result = {'version': 1, 'duration': clip['duration'], 'targets': list(targets.values()), + 'curves': curves, 'events': clip['events'], 'frames': frames, + 'source': {'controller': source['controller']['source'], 'clip': clip['source'], + 'animatorId': animator_id, 'clipIndex': motion['clipIndex'], + 'accounting': clip['accounting']}} + relative = 'photo-shot/presentation.json' + path = write_json(output / relative, result) + return {'document': relative, 'kind': 'photo-presentation', + 'documentSha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'curves': len(curves), 'frames': len(frames)} diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index ef0172d..ed0dd62 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -142,6 +142,7 @@ # The mission cells sit in Resources folders and are instantiated from # their lists' serialized cell prefab reference (see POINTER_PREFABS). "missioncell": ("MysekaiMissionScreenCell", "MysekaiMissionRewardCell"), + "photocell": ("UIPartsPhotoFrameListThumbnail",), } # Screen layers and dialogs the JP UI census maps to but the product does not @@ -288,7 +289,11 @@ # repeats its page, whose pages each hold a copy of the content beside the list's # cell instance, adds a fourth item, the node's path under the page, and builds # the prefab from the page's own nodes (see follow_page_prefab_reference). -POINTER_PREFABS = {**CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS} +POINTER_PREFABS = { + **CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS, + "UIPartsPhotoFrameListThumbnail": ( + "ScreenLayerMysekaiPhotoShot", "Sekai.FixedSizeReuseScroller", "prefab"), +} POINTER_PAGE_PREFABS = {**MISSION_PAGE_POINTER_PREFABS} # Prefabs -> (holder, the holder's list field, the cell's name): the field names @@ -794,6 +799,7 @@ def decode_screen_layer_data_list(r: talk.Reader) -> dict: **FIELD_HUD_CN_DECODERS, "UnityEngine.UI.Toggle": decode_unity_toggle, "Sekai.UI.CustomRawImage": decode_raw_image_base, + "UnityEngine.UI.RawImage": decode_raw_image_base, "Sekai.UI.CustomToggle": decode_custom_toggle, "Sekai.UI.CustomIndexToggleGroup": decode_custom_index_toggle_group, "Sekai.MenuDialogCell": decode_menu_dialog_cell, @@ -3490,6 +3496,10 @@ def digest(path): runtime_textures[alias] = sprite["image"] runtime_sprites[alias] = sprite path = write_json(out / family / f"{prefab}.json", document) + if prefab == "ScreenLayerMysekaiPhotoShot": + from ui.photo_presentation import export_presentation + presentation = export_presentation(document, master_path, bundles_root, out) + rows.append({**presentation, **identity}) monos, decoded, raw = _document_counts(document) closure = _closure_of(env, file_name) row = { diff --git a/src/ui/talk.py b/src/ui/talk.py index c1af96a..312da91 100644 --- a/src/ui/talk.py +++ b/src/ui/talk.py @@ -58,7 +58,7 @@ def resolve_script_class(obj, mono_index: dict) -> str: raw = obj.get_raw_data() if len(raw) < 28: return "?" - fid, pid = struct.unpack(" Date: Tue, 27 Oct 2026 06:40:02 +0800 Subject: [PATCH 09/15] feat(pipeline): publish independent resources and editor inputs --- docs/site-edit-pick.md | 34 ++++ src/cli/legacy.py | 13 ++ src/fixtures/master_slice.py | 7 + src/sites/edit_pick.py | 227 +++++++++++++++++++++++ src/ui/editor_layout.py | 16 +- src/ui/prefab_layout.py | 2 + tests/test_editor_environment_layout.py | 25 +++ tests/test_fixture_master_slice.py | 17 ++ tests/test_font_publication.py | 9 +- tests/test_resource_publication.py | 140 ++++++++++++++ tests/test_s3_deployment.py | 30 ++- tests/test_site_edit_pick.py | 189 +++++++++++++++++++ tools/deployment/README.md | 22 +++ tools/deployment/S3-MIGRATION.md | 7 + tools/deployment/prepare_fonts.py | 2 +- tools/deployment/prepare_resources.py | 88 +++++++++ tools/deployment/prepare_translations.py | 2 +- tools/deployment/resource_publication.py | 150 +++++++++++++++ tools/deployment/upload_oss.py | 19 +- tools/deployment/upload_s3.py | 83 +++++---- 20 files changed, 1038 insertions(+), 44 deletions(-) create mode 100644 docs/site-edit-pick.md create mode 100644 src/sites/edit_pick.py create mode 100644 tests/test_editor_environment_layout.py create mode 100644 tests/test_resource_publication.py create mode 100644 tests/test_site_edit_pick.py create mode 100644 tools/deployment/prepare_resources.py create mode 100644 tools/deployment/resource_publication.py diff --git a/docs/site-edit-pick.md b/docs/site-edit-pick.md new file mode 100644 index 0000000..f93c8ef --- /dev/null +++ b/docs/site-edit-pick.md @@ -0,0 +1,34 @@ +# Site editor picking surface + +The site editor raycasts the scene colliders tagged `RaycastTargetGroup`. +Export them from the scene bundle and its matching client APK: + +```sh +moly export site-edit-pick \ + --apk client.apk \ + --bundle decrypted/mysekai__site__field__home \ + --out assets/site/edit-pick/home.json +``` + +`--apk` can be repeated for split APKs. The APK must supply +`assets/bin/Data/globalgamemanagers`; its `TagManager.tags` determines the numeric +tag. Use the APK from the same client version as the bundle. The exporter records +the source bundle hash, TagManager file hash and object identifiers; these prove +the bytes supplied but do not infer whether two independently supplied inputs +belong to the same release. + +The version 1 sidecar retains each tagged MeshCollider's enabled, trigger and +convex flags, GameObject layer, root-to-leaf active flags and local TRS, authored +vertex positions and triangle winding. Coordinates stay Unity left-handed, +Y-up; no scene placement, reflection, visual geometry or navigation mesh is +substituted. Triangle indices include each submesh's base vertex. + +Version 1 requires MeshCollider triangle geometry and local pointers in the +collider's serialized file. Missing tags, unresolved pointers, other collider +types, unsupported topology or invalid geometry fail before replacing an +existing output. A disabled collider or inactive ancestor remains represented +with its source flag; the runtime decides whether it can be hit. + +Add the resulting `site/edit-pick/.json` to the resource manifest and use +the existing resource publication workflow. This command only writes the +explicit output file; it does not change a source snapshot or upload resources. diff --git a/src/cli/legacy.py b/src/cli/legacy.py index 71bdfe1..8634066 100644 --- a/src/cli/legacy.py +++ b/src/cli/legacy.py @@ -450,6 +450,14 @@ def main(argv=None): help="directory containing declared effect, mesh, texture and shader dependencies") fpe.add_argument("--out-dir", required=True, help="effect directory, normally /fixture-put-effects") + sep = sub.add_parser("site-edit-pick", + help="export tagged editor raycast colliders in authored Unity coordinates") + sep.add_argument("--bundle", required=True, + help="one mysekai__site__field__ source bundle") + sep.add_argument("--apk", action="append", required=True, + help="matching client APK containing TagManager; repeat for split APKs") + sep.add_argument("--out", required=True, + help="output sidecar, normally /site/edit-pick/.json") ssw = sub.add_parser("site-sound-wave", help="resolve MysekaiSiteView._soundWaveMeshFilter (the footstep " "surface) of site scene packages: node, mesh and vertex colours") @@ -1501,6 +1509,11 @@ def main(argv=None): bundle_root=args.bundle_root), ensure_ascii=False)) return 0 + if args.cmd == "site-edit-pick": + from sites.edit_pick import extract_site_edit_pick + print(json.dumps(extract_site_edit_pick(args.bundle, args.apk, args.out), + ensure_ascii=False)) + return 0 if args.cmd == "site-sound-wave": from sites.sound_wave import extract_site_sound_wave result = extract_site_sound_wave(args.bundle, args.out, bundle_root=args.bundle_root) diff --git a/src/fixtures/master_slice.py b/src/fixtures/master_slice.py index 20a1b3e..8eb53ca 100644 --- a/src/fixtures/master_slice.py +++ b/src/fixtures/master_slice.py @@ -108,6 +108,12 @@ def export_fixture_master_slice( sub_genre = row.get("mysekaiFixtureSubGenreId") if sub_genre is not None and type(sub_genre) is not int: raise ValueError(f"row id={row['id']!r}: sub genre id must be an integer or null") + costs = {field: row.get(field) for field in ("firstPutCost", "secondPutCost")} + for field, value in costs.items(): + # Older clients do not expose cost limits. Missing stays null; + # consumers cannot turn an absent cost into a free placement. + if value is not None and (type(value) is not int or value < 0): + raise ValueError(f"row id={row['id']!r}: {field} must be a nonnegative integer or null") grid = row["gridSize"] for axis in ("width", "depth", "height"): if axis not in grid: @@ -130,6 +136,7 @@ def export_fixture_master_slice( "gridDepth": int(grid["depth"]), "gridHeight": int(grid["height"]), "putSoundId": put_sound, + **costs, } ) out.sort(key=lambda record: record["id"]) diff --git a/src/sites/edit_pick.py b/src/sites/edit_pick.py new file mode 100644 index 0000000..5235acc --- /dev/null +++ b/src/sites/edit_pick.py @@ -0,0 +1,227 @@ +"""Source collision surfaces used by the site's editor raycast. + +The editor selects colliders by the RaycastTargetGroup tag. Its numeric value +belongs to the client build's TagManager, so the corresponding APK is an +explicit input. Model names and the visible or navigation meshes do not select +this surface. The small runtime sidecar keeps Unity's authored coordinates, +triangle winding and root-to-leaf transform chain without a glTF conversion. +""" +from __future__ import annotations + +import json +import math +import os +import tempfile +from pathlib import Path + +from UnityPy.helpers.MeshHelper import MeshHandler + +from core.assets.packages import PackageStore, require_paths +from core.unity import configure_fallback_unity_version +from sites.geometry import Graph +from ui.player_source import ApkPlayerData + +TAG = "RaycastTargetGroup" +CUSTOM_TAG_BASE = 20000 +SCENE_PREFIX = "mysekai__site__field__" +COORDINATE_CONTRACT = "unity-lh-y-up-authored" + + +def _tag_index(tree): + tags = tree.get("tags") + if not isinstance(tags, list): + raise ValueError("TagManager has no tags array") + matches = [i for i, tag in enumerate(tags) if tag == TAG] + if len(matches) != 1: + raise ValueError(f"expected one {TAG} tag, found {len(matches)}") + return CUSTOM_TAG_BASE + matches[0] + + +def _tag_manager(apks): + source = ApkPlayerData(apks) + player = source.environment() + player.add(["globalgamemanagers"], follow=False) + managers = [obj for obj in player.env.objects if obj.type.name == "TagManager"] + if len(managers) != 1: + raise ValueError(f"expected one TagManager, found {len(managers)}") + manager = managers[0] + return _tag_index(manager.read_typetree()), { + "file": "globalgamemanagers", + "fileSha256": source.sha256("globalgamemanagers"), + "objectId": str(manager.path_id), + } + + +def _local_target(record, pointer, kind): + pointer = pointer or {} + if pointer.get("m_FileID", 0): + raise ValueError(f"edit-pick v1 requires {kind} in the collider's file") + path_id = pointer.get("m_PathID", 0) + if not path_id or record.kinds.get(path_id) != kind: + raise ValueError(f"unresolved {kind} pointer: {path_id}") + return path_id + + +def _vector(tree, field, axes): + try: + result = [float(tree[field][axis]) for axis in axes] + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"missing or invalid {field}") from exc + if not all(math.isfinite(value) for value in result): + raise ValueError(f"non-finite {field}") + return result + + +def _flag(tree, field): + value = tree.get(field) + if value not in (0, 1) or not isinstance(value, (int, bool)): + raise ValueError(f"missing or invalid {field}") + return bool(value) + + +def _chain(record, graph, game_object): + transform_id = graph.transform_of.get(game_object) + if transform_id is None: + raise ValueError(f"GameObject {game_object} has no Transform") + chain, seen = [], set() + while transform_id: + if transform_id in seen: + raise ValueError("transform parent chain loops") + seen.add(transform_id) + if record.kinds.get(transform_id) != "Transform": + raise ValueError(f"unresolved Transform: {transform_id}") + transform = record.tree(transform_id) + owner = _local_target(record, transform.get("m_GameObject"), "GameObject") + chain.append({ + "position": _vector(transform, "m_LocalPosition", "xyz"), + "rotation": _vector(transform, "m_LocalRotation", "xyzw"), + "scale": _vector(transform, "m_LocalScale", "xyz"), + "active": _flag(record.tree(owner), "m_IsActive"), + }) + parent = transform.get("m_Father") + if not isinstance(parent, dict) or "m_PathID" not in parent: + raise ValueError("Transform has no authored parent pointer") + if parent.get("m_FileID", 0): + raise ValueError("edit-pick v1 requires parent transforms in the collider's file") + transform_id = parent.get("m_PathID", 0) + return list(reversed(chain)) + + +def _mesh_geometry(mesh): + handler = MeshHandler(mesh) + handler.process() + positions = [[float(value) for value in vertex[:3]] for vertex in handler.m_Vertices] + if (not positions or len(positions) != handler.m_VertexCount + or any(len(vertex) != 3 or not all(math.isfinite(v) for v in vertex) + for vertex in positions)): + raise ValueError("collider mesh has missing or invalid vertices") + indices = handler.m_IndexBuffer + if indices is None: + raise ValueError("collider mesh has no indices") + width = 2 if handler.m_Use16BitIndices else 4 + triangles = [] + for submesh in mesh.m_SubMeshes or []: + if submesh.topology != 0: + raise ValueError("edit-pick v1 requires authored triangle topology") + start, count = submesh.firstByte, submesh.indexCount + if start < 0 or start % width or count < 0 or count % 3: + raise ValueError("invalid triangle submesh range") + start //= width + if start + count > len(indices): + raise ValueError("triangle submesh exceeds the index buffer") + # Unity's indices are relative to baseVertex. MeshHandler.get_triangles + # does not apply that offset, so read the authored triangle ranges here. + base = getattr(submesh, "baseVertex", None) or 0 + for offset in range(start, start + count, 3): + triangle = [int(index) + base for index in indices[offset:offset + 3]] + if any(index < 0 or index >= len(positions) for index in triangle): + raise ValueError("triangle index outside the vertex buffer") + triangles.append(triangle) + if not triangles: + raise ValueError("collider mesh has no triangles") + return positions, triangles + + +def _colliders(package, tag_index): + colliders = [] + for record in sorted(package.files, key=lambda item: item.archive): + graph = None + for path_id, kind in sorted(record.kinds.items()): + if not (kind.endswith("Collider") or kind == "CharacterController"): + continue + collider = record.tree(path_id) + owner = _local_target(record, collider.get("m_GameObject"), "GameObject") + game_object = record.tree(owner) + tag = game_object.get("m_Tag") + if isinstance(tag, bool) or not isinstance(tag, int) or tag < 0: + raise ValueError("missing or invalid GameObject tag") + if tag != tag_index: + continue + if kind != "MeshCollider": + raise ValueError(f"unsupported {TAG} component: {kind}") + mesh_id = _local_target(record, collider.get("m_Mesh"), "Mesh") + if graph is None: + graph = Graph(record) + positions, triangles = _mesh_geometry(record.objects[mesh_id].read()) + layer = game_object.get("m_Layer") + if isinstance(layer, bool) or not isinstance(layer, int) or not 0 <= layer < 32: + raise ValueError("missing or invalid GameObject layer") + colliders.append({ + "tag": TAG, "tagIndex": tag_index, + "enabled": _flag(collider, "m_Enabled"), + "isTrigger": _flag(collider, "m_IsTrigger"), + "convex": _flag(collider, "m_Convex"), "layer": layer, + "chain": _chain(record, graph, owner), + "positions": positions, "triangles": triangles, + "source": {"file": record.archive, "componentId": str(path_id), + "gameObjectId": str(owner), "meshId": str(mesh_id)}, + }) + if not colliders: + raise ValueError(f"scene has no {TAG} colliders") + return colliders + + +def extract_site_edit_pick(bundle, apks, out): + """Export one scene sidecar from its source bundle and matching client APKs. + + All pointers used by schema v1 must stay in the collider's serialized file; + otherwise its single-file provenance would be incomplete, so fail rather + than publish a partial surface. Other scene dependencies are not rendered + or loaded. The explicit output file is replaced atomically after success. + """ + configure_fallback_unity_version() + bundle, apks, out = Path(bundle), [Path(apk) for apk in apks], Path(out) + require_paths([bundle]) + name = bundle.name + site = name.removeprefix(SCENE_PREFIX) + if not name.startswith(SCENE_PREFIX) or not site: + raise ValueError(f"bundle must be named {SCENE_PREFIX}") + if out.resolve() in {path.resolve() for path in [bundle, *apks]}: + raise ValueError("output must not replace an input") + tag_index, tag_source = _tag_manager(apks) + store = PackageStore([bundle]) + package = store.package(name) + if package is None: + raise ValueError(f"scene package not loaded: {name}") + document = { + "version": 1, "coordinateContract": COORDINATE_CONTRACT, + "site": site, "package": name, + "bundleSha256": store.source_receipt(name)["sha256"], + "colliders": _colliders(package, tag_index), "tagManagerSource": tag_source, + } + payload = json.dumps(document, ensure_ascii=False, separators=(",", ":"), + allow_nan=False).encode("utf-8") + out.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=out.parent, prefix=f".{out.name}.", + delete=False) as handle: + temporary = Path(handle.name) + handle.write(payload) + os.replace(temporary, out) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return {"output": str(out), "site": site, "colliders": len(document["colliders"]), + "vertices": sum(len(row["positions"]) for row in document["colliders"]), + "triangles": sum(len(row["triangles"]) for row in document["colliders"])} diff --git a/src/ui/editor_layout.py b/src/ui/editor_layout.py index a940ac2..fe4bd30 100644 --- a/src/ui/editor_layout.py +++ b/src/ui/editor_layout.py @@ -8,7 +8,7 @@ "ScreenLayerSiteEditMode", "MysekaiEditSaveConfirmationDialog", "SiteEditListSelectorCell", "SiteEditOutDoorContentList", "SiteEditFloorContentList", "SiteEditWallContentList", "SiteEnvironmentContentList", "RoadFixtureContentList", - "FixtureSelectCell", "UIPartsLeftTabListMysekaiContentCell", + "FixtureSelectCell", "SiteEnvironmentSelectCell", "UIPartsLeftTabListMysekaiContentCell", "ScreenLayerHeader", ) @@ -177,6 +177,18 @@ def decode_left_tab_controller(r): return d +def decode_site_environment_cell(r): + """ListViewItem fields, then SiteEnvironmentSelectCell's two references.""" + return {"sizeX": r.f32(), "sizeY": r.f32(), + **refs(r, ("_thumbnail", "_selected"))} + + +def decode_site_environment_thumbnail(r): + """UIPartsThumbnail fields, then its environment button and detector.""" + from ui.bgm_select_layout import decode_thumbnail + return {**decode_thumbnail(r), **refs(r, ("button", "clickDetector"))} + + def decode_fixture_cell(r): d = {"sizeX": r.f32(), "sizeY": r.f32()} d.update(refs(r, ("thumbnail", "selected", "_inPlacedLabel", "_missionLabel"))) @@ -301,6 +313,8 @@ def decode_layer_data(r): "Sekai.UI.UITextureLoader": decode_texture_loader, "Sekai.UI.UIPartsLeftTabListController": decode_left_tab_controller, "Sekai.UI.FixtureSelectCell": decode_fixture_cell, + "Sekai.UI.SiteEnvironmentSelectCell": decode_site_environment_cell, + "Sekai.Mysekai.UIPartsSiteEnvironmentThumbnail": decode_site_environment_thumbnail, "Sekai.UI.UIPartsIconInfoView": lambda r: refs(r, ("infoTextRoot", "infoText", "lockedObject")), "Sekai.UI.UIPartsLeftTabList": lambda r: refs(r, ("scroller", "topScrollableIcon", "bottomScrollableIcon", "backPanelTransform")), "Sekai.UI.UIPartsLeftTabSubList": lambda r: refs(r, ("scroller", "topScrollableIcon", "bottomScrollableIcon")), diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index ed0dd62..13a66eb 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -290,6 +290,8 @@ # cell instance, adds a fourth item, the node's path under the page, and builds # the prefab from the page's own nodes (see follow_page_prefab_reference). POINTER_PREFABS = { + "SiteEnvironmentSelectCell": ( + "SiteEnvironmentContentList", "Sekai.UI.ListView", "listViewItemPrefab"), **CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS, "UIPartsPhotoFrameListThumbnail": ( "ScreenLayerMysekaiPhotoShot", "Sekai.FixedSizeReuseScroller", "prefab"), diff --git a/tests/test_editor_environment_layout.py b/tests/test_editor_environment_layout.py new file mode 100644 index 0000000..71e49f6 --- /dev/null +++ b/tests/test_editor_environment_layout.py @@ -0,0 +1,25 @@ +"""The environment cell follows the same serialized list reference as the client.""" +import struct + +from ui import editor_layout, prefab_layout, talk + + +def test_environment_cell_keeps_list_metrics_and_the_two_references(): + reader = talk.Reader(struct.pack(" maximum: + raise ValueError('store object exceeds runtime budget') + return resolved, data + + +def catalog_closure(store, catalog_id): + """Verify the catalogue, package graph, source identity and all stored blobs.""" + store = Path(store).resolve(strict=True) + if not isinstance(catalog_id, str) or not HASH.fullmatch(catalog_id): + raise ValueError('invalid resource catalogue hash') + name = f'catalogs/{catalog_id}.json' + path, raw = read_store_file(store, name, MAX_RUNTIME_DOCUMENT_BYTES) + catalog = strict_json(raw) + if (digest(raw) != catalog_id or catalog.get('schema') != 'moly-asset-packs/2' + or catalog.get('region') not in {'jp', 'cn', 'tw', 'en', 'kr'} + or not re.fullmatch(r'\d+\.\d+\.\d+', catalog.get('version', '')) + or raw != canonical_bytes(catalog)): + raise ValueError('invalid resource catalogue identity') + if public_release_provenance(catalog.get('provenance')) != catalog.get('provenance'): + raise ValueError('resource catalogue provenance is not portable') + errors, info = verify_catalog_data(catalog, store) + if errors: + raise ValueError('invalid resource catalogue closure: ' + '; '.join(errors[:8])) + objects = {name: {'source': path, 'sha256': catalog_id, 'bytes': len(raw)}} + entries = {} + for package in catalog['packages']: + name = package['manifest'] + path, raw = read_store_file(store, name, MAX_RUNTIME_DOCUMENT_BYTES) + if digest(raw) != Path(name).stem: + raise ValueError('package changed during resource verification') + objects[name] = {'source': path, 'sha256': digest(raw), 'bytes': len(raw)} + for entry in strict_json(raw)['entries']: + # The v2 native and Web readers share this representation contract. + # The archive verifier also accepts legacy Brotli forms, but those + # cannot be published as a usable runtime catalogue. + if (entry['codec'] not in {'identity', 'gzip'} + or entry['http_encoding'] != 'identity' or entry['xf'] is not None): + raise ValueError('resource catalogue uses a representation unsupported by native/Web runtime') + entries[entry['path']] = entry + name = 'blobs/' + entry['blob'] + path, raw = read_store_file(store, name, MAX_RUNTIME_BLOB_BYTES) + if len(raw) != entry['blob_bytes'] or digest(raw) != entry['blob_sha256']: + raise ValueError('blob changed during resource verification') + objects[name] = {'source': path, 'sha256': entry['blob_sha256'], 'bytes': entry['blob_bytes']} + source_entry = entries.get('source.json') + provenance = catalog.get('provenance', {}).get('runtime_source') + if source_entry is None or not isinstance(provenance, dict): + raise ValueError('resource catalogue requires its runtime source document') + _, source_blob = read_store_file(store, 'blobs/' + source_entry['blob'], MAX_RUNTIME_BLOB_BYTES) + source_raw = _decode_content(source_blob, source_entry['codec'], source_entry['http_encoding'], source_entry['bytes']) + if strict_json(source_raw) != provenance: + raise ValueError('catalogue provenance differs from the packaged source document') + source_identity = provenance.get('source', {}) + if (source_identity.get('region') != catalog['region'] + or source_identity.get('appVersion') != catalog['version']): + raise ValueError('packaged source region/version differs from its resource catalogue') + identity = {'path': f'asset-store/catalogs/{catalog_id}.json', 'sha256': catalog_id, + 'bytes': objects[f'catalogs/{catalog_id}.json']['bytes'], + 'region': catalog['region'], 'version': catalog['version'], + 'provenance': catalog['provenance']} + return identity, objects, entries, info + + +def validate_upload(doc): + """Revalidate actual sources and representations before selecting any upload.""" + from prepare_deployment import content_type + identity = doc['resourceCatalog'] + root_rows = [row for row in doc['files'] if row['path'] == identity['path']] + if len(root_rows) != 1: + raise ValueError('resource catalogue row is absent or duplicated') + raw_path = Path(root_rows[0]['sourceRaw']).absolute() + if raw_path.parent.name != 'catalogs' or raw_path.name != identity['sha256'] + '.json': + raise ValueError('resource catalogue source is not its addressed store object') + store = raw_path.parent.parent.resolve(strict=True) + actual, objects, _, _ = catalog_closure(store, identity['sha256']) + if actual != identity: + raise ValueError('resource catalogue identity or provenance changed') + rows = {row['path']: row for row in doc['files']} + expected = {'asset-store/' + name for name in objects} + if rows.keys() != expected: + raise ValueError('resource publication object closure mismatch') + url_prefix = doc.get('urlPrefix') + if (not isinstance(url_prefix, str) or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for part in url_prefix.split('/')) + or any(ord(c) < 32 or ord(c) == 127 or c in '\\%?#:' for c in url_prefix)): + raise ValueError('invalid resource URL prefix') + checked = set() + for name, expected in objects.items(): + logical = 'asset-store/' + name + row = rows[logical] + if (Path(row['sourceRaw']).absolute() != expected['source'] + or row['decodedSha256'] != expected['sha256'] or row['decodedBytes'] != expected['bytes'] + or row['bytes'] > expected['bytes'] + or row['contentType'] != content_type(logical, expected['source']) + or row['cacheControl'] != CACHE_CONTROL or row.get('urlPath') != url_prefix + logical): + raise ValueError('resource object metadata differs from its catalogue') + for alias in [row.get('source'), *([row['staging']] if 'staging' in row else [])]: + if not isinstance(alias, str) or not alias: + raise ValueError('invalid resource object source') + path = Path(alias).resolve(strict=True) + key = (path, row['sha256'], row['bytes'], expected['sha256'], expected['bytes'], row['contentEncoding']) + if key in checked: + continue + with path.open('rb') as stream: + body = stream.read(row['bytes'] + 1) + if len(body) != row['bytes'] or digest(body) != row['sha256']: + raise ValueError('local resource representation changed') + # The HTTP representation expands to the stored blob/package bytes. + # A pack codec, if present, is subsequently decoded by the runtime. + encoding = row['contentEncoding'] + decoded = _decode_content(body, 'gzip' if encoding == 'gzip' else 'identity', + 'br' if encoding == 'br' else 'identity', expected['bytes']) + if len(decoded) != expected['bytes'] or digest(decoded) != expected['sha256']: + raise ValueError('resource HTTP representation differs from the addressed object') + checked.add(key) diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 2723ec2..7e5c9c5 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -247,6 +247,15 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) snapshots = set(doc.get('snapshots', [])) translation = doc.get('translation') font_catalog = doc.get('fontCatalog') + resource_catalog = doc.get('resourceCatalog') + if resource_catalog is not None: + if (not isinstance(resource_catalog, dict) or snapshots or translation is not None + or font_catalog is not None or doc.get('release') is not None + or not SHA256.fullmatch(resource_catalog.get('sha256', '')) + or resource_catalog.get('path') != f"asset-store/catalogs/{resource_catalog['sha256']}.json"): + raise SafeFailure('invalid_resource_catalogue_publication') + if args.scope != 'stable' or args.limit is not None: + raise SafeFailure('resource_catalogue_requires_complete_stable_scope') if font_catalog is not None: if (not isinstance(font_catalog, dict) or snapshots or translation is not None or font_catalog.get('path') != 'fonts/latest.json' @@ -262,7 +271,7 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) translation_base = translation['path'].removesuffix('latest.json') else: translation_base = None - if (not snapshots and translation is None and font_catalog is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): + if (not snapshots and translation is None and font_catalog is None and resource_catalog is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): raise SafeFailure('invalid_snapshot_identities') if args.scope == 'release' and (args.release_id != doc.get('release') or not IDENTIFIER.fullmatch(args.release_id or '')): raise SafeFailure('explicit_release_id_must_match_manifest') @@ -280,6 +289,8 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected, seen = [], set() for row in doc.get('files', []): logical = row.get('path', '') + if resource_catalog is not None and not re.fullmatch(r'asset-store/(?:(?:catalogs|packages)/[a-f0-9]{64}\.json|blobs/[a-f0-9]{2}/[a-f0-9]{64}\.(?:bin|gzz|brz|br))', logical): + raise SafeFailure('resource_catalogue_contains_unrelated_object') if font_catalog is not None and not re.fullmatch(r'fonts/(?:latest\.json|revisions/[a-f0-9]{64}\.json|objects/[a-f0-9]{64}\.(?:ttf|txt))', logical): raise SafeFailure('font_manifest_contains_unrelated_object') if translation_base and (not logical.startswith(translation_base) or not re.fullmatch( @@ -334,6 +345,12 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) raise SafeFailure('no_objects_selected') if font_catalog is not None: validate_font_publication(doc) + if resource_catalog is not None: + try: + from resource_publication import validate_upload + validate_upload(doc) + except (OSError, ValueError, KeyError, TypeError) as error: + raise SafeFailure('invalid_resource_catalogue_closure', errorType=type(error).__name__) from None if translation is not None: roots = [row for row in doc['files'] if row['path'] == translation['path']] if len(roots) != 1 or roots[0]['decodedSha256'] != translation['sha256']: diff --git a/tools/deployment/upload_s3.py b/tools/deployment/upload_s3.py index 69bc132..5730cc4 100644 --- a/tools/deployment/upload_s3.py +++ b/tools/deployment/upload_s3.py @@ -295,11 +295,13 @@ def selection(args): return doc, rows -def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys=None, absent_keys=None): +def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys=None, absent_keys=None, + capacity_bytes=None): """Current encoded object bytes, all bucket prefixes; never decoded lengths. `absent_keys`, when given, receives the selected keys the complete listing does not contain.""" + capacity = CAPACITY if capacity_bytes is None else capacity_bytes incomplete = [] try: if client.get_bucket_versioning(Bucket=target['bucket']).get('Status'): @@ -316,30 +318,40 @@ def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys incomplete.append('multipart_storage_unverified') used = count = 0 listed_keys = set() - continuation = None - for _ in range(10000): - response = client.list_objects_v2(Bucket=target['bucket'], MaxKeys=1000, - **({'ContinuationToken': continuation} if continuation else {})) - if type(response.get('IsTruncated')) is not bool: - raise SafeFailure('bucket_listing_completeness_unverified') - for item in response.get('Contents', []): - size = item.get('Size') - if type(size) is not int or size < 0: - raise SafeFailure('invalid_bucket_object_size') - used += size - count += 1 - key = item.get('Key') - if not isinstance(key, str) or not key: - raise SafeFailure('invalid_bucket_object_key') - listed_keys.add(key) - if not response.get('IsTruncated'): - break - token = response.get('NextContinuationToken') - if not token or token == continuation: - raise SafeFailure('invalid_bucket_listing_cursor') - continuation = token - else: - raise SafeFailure('bucket_listing_page_limit') + # Without an operator capacity budget, only the selected namespaces need + # inventory. Unrelated historic assets do not affect this upload's delta. + prefixes = [None] if capacity else sorted({ + row['objectKey'].rsplit('/', 1)[0] + '/' if '/' in row['objectKey'] else row['objectKey'] + for row in rows + }) + for prefix in prefixes: + continuation = None + seen_tokens = set() + for _ in range(10000): + response = client.list_objects_v2(Bucket=target['bucket'], MaxKeys=1000, + **({'Prefix': prefix} if prefix is not None else {}), + **({'ContinuationToken': continuation} if continuation else {})) + if type(response.get('IsTruncated')) is not bool: + raise SafeFailure('bucket_listing_completeness_unverified') + for item in response.get('Contents', []): + size, key = item.get('Size'), item.get('Key') + if type(size) is not int or size < 0: + raise SafeFailure('invalid_bucket_object_size') + if not isinstance(key, str) or not key: + raise SafeFailure('invalid_bucket_object_key') + if key not in listed_keys: + used += size + count += 1 + listed_keys.add(key) + if not response.get('IsTruncated'): + break + token = response.get('NextContinuationToken') + if not token or token in seen_tokens: + raise SafeFailure('invalid_bucket_listing_cursor') + seen_tokens.add(token) + continuation = token + else: + raise SafeFailure('bucket_listing_page_limit') additional = transfer = existing_count = 0 conflicts = [] conflict_count = 0 @@ -367,13 +379,15 @@ def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys # reclamation of an old representation during the upload. additional += row['bytes'] transfer += row['bytes'] - return {'capacityBytes': CAPACITY, 'currentObjectBytes': used, 'objects': count, + return {'capacityBytes': capacity or None, 'currentObjectBytes': used if capacity else None, + 'objects': count if capacity else None, 'listedObjectBytes': used, 'listedObjects': count, 'additionalBytesUpperBound': additional, 'transferBytes': transfer, - 'matchingSelectedObjects': existing_count, 'projectedBytesUpperBound': used + additional, + 'matchingSelectedObjects': existing_count, 'projectedBytesUpperBound': used + additional if capacity else None, 'immutableConflictCount': conflict_count, 'immutableConflicts': conflicts, 'complete': not incomplete, 'incompleteReasons': incomplete, - 'withinCapacity': not incomplete and used + additional <= CAPACITY, - 'scope': 'whole-bucket encoded object payload; requires an exclusive publisher; not a quota reservation'} + 'withinCapacity': not incomplete and (not capacity or used + additional <= capacity), + 'scope': ('whole-bucket encoded object payload; requires an exclusive publisher; not a quota reservation' + if capacity else 'selected namespaces; incremental accounting; operator disabled capacity budget')} def previous_digest(client, target, key, existing): @@ -625,12 +639,16 @@ def parse_args(argv=None): parser.add_argument('--inspect', action='store_true', help='Read-only whole-bucket capacity and object inspection') parser.add_argument('--execute', action='store_true') parser.add_argument('--exclusive-writer', action='store_true', help='Confirm no concurrent publisher can consume the planned bucket capacity') + parser.add_argument('--capacity-bytes', type=int, default=CAPACITY, + help='Operator storage budget in bytes (default: 30 GiB); 0 explicitly disables the budget and inventories selected namespaces') return parser.parse_args(argv) def main(argv=None): global THROTTLE args = parse_args(argv) + if args.capacity_bytes < 0: + raise SafeFailure('capacity_bytes_must_be_nonnegative') if args.max_bytes_per_second is not None: if args.max_bytes_per_second < UPLOAD_CHUNK: raise SafeFailure('max_bytes_per_second_below_one_chunk', minimum=UPLOAD_CHUNK) @@ -646,7 +664,7 @@ def main(argv=None): copy_sources = load_copy_sources(args.copy_source_manifest, args.object_prefix) copyable = [row for row in rows if copy_sources.get(content_identity(row), row['objectKey']) != row['objectKey']] common.emit('selection', destination=target, dryRun=not args.execute, objects=len(rows), - selectedEncodedBytes=sum(row['bytes'] for row in rows), capacityBytes=CAPACITY, + selectedEncodedBytes=sum(row['bytes'] for row in rows), capacityBytes=args.capacity_bytes or None, receipt=str(receipt_path(args, identity)), deleteObjects=False, changeAcl=False, copyCandidates=len(copyable), copyCandidateBytes=sum(row['bytes'] for row in copyable)) if not args.inspect and not args.execute: @@ -674,12 +692,13 @@ def main(argv=None): require_copy_conditions(client) absent_keys = set() plan = inspect_capacity(client, target, rows, mutable=args.scope == 'mutable', - known_existing_keys=known_existing_keys, absent_keys=absent_keys) + known_existing_keys=known_existing_keys, absent_keys=absent_keys, + capacity_bytes=args.capacity_bytes) common.emit('capacity-plan', **plan) if not args.execute: return 0 if not plan['withinCapacity']: - raise SafeFailure('capacity_unverified_or_exceeded', capacityBytes=CAPACITY) + raise SafeFailure('capacity_unverified_or_exceeded', capacityBytes=args.capacity_bytes or None) if plan['immutableConflictCount']: raise SafeFailure('immutable_object_conflicts', count=plan['immutableConflictCount']) return execute(client, target, rows, args, identity, absent_keys, copy_sources) From 3f2ba35fcf42ff11368cbc95ffc51ec0bb5e2f66 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:25:07 +0800 Subject: [PATCH 10/15] fix(publication): verify independent stage and game releases --- tests/test_independent_releases.py | 134 +++++++++++++++++++++++++ tools/deployment/prepare_deployment.py | 87 +++++++++++++--- tools/deployment/upload_oss.py | 32 +++++- tools/head-portraits/export.mjs | 6 +- 4 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 tests/test_independent_releases.py diff --git a/tests/test_independent_releases.py b/tests/test_independent_releases.py new file mode 100644 index 0000000..df51eaf --- /dev/null +++ b/tests/test_independent_releases.py @@ -0,0 +1,134 @@ +"""Independent stage/game engines must both be in the verified release closure.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +TOOLS = Path(__file__).parents[1] / 'tools/deployment' +CONTRACT = 'moly-rh-y-up-reflect-x-v1' + + +def load(name): + spec = importlib.util.spec_from_file_location('independent_' + name, TOOLS / (name + '.py')) + tool = importlib.util.module_from_spec(spec) + spec.loader.exec_module(tool) + return tool + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + +@pytest.fixture +def publication(tmp_path): + root = tmp_path / 'publication' + snapshot = {'id': 'cn-synthetic', 'region': 'cn', 'version': '6.0.0', + 'assets': '/moly/snapshots/cn-synthetic/assets/', 'packs': False, 'assetCatalog': None} + manifest = {'publisher': 'moly-release-artifact-v1', 'schemaVersion': 2, + 'release': {'id': 'stage-synthetic', 'coordinateContract': CONTRACT}, 'snapshots': [snapshot]} + write(root / 'manifest.json', manifest) + game = {'publisher': 'moly-release-artifact-v1', 'schemaVersion': 1, 'prefix': '/moly/', + 'release': {'id': 'game-synthetic', 'module': '/moly/releases/game-synthetic/game-boot.mjs', + 'abi': 2, 'coordinateContract': CONTRACT}, 'snapshots': [snapshot]} + write(root / 'game/manifest.json', game) + for release, module in [('stage-synthetic', 'embed.mjs'), ('game-synthetic', 'game-boot.mjs')]: + base = root / 'releases' / release + base.mkdir(parents=True) + raw = f'export const name = "{release}";'.encode() + (base / module).write_bytes(raw) + write(base / 'integrity.json', {'schemaVersion': 1, 'releaseId': release, 'contractVersion': 2, + 'coordinateContract': CONTRACT, 'hashes': {module: hashlib.sha256(raw).hexdigest()}}) + (root / 'cache-worker.mjs').write_text('// synthetic worker') + assets = root / 'snapshots/cn-synthetic/assets' + for name, value in [('mysekai-fixtures.json', {'region': 'cn', 'gameVersion': '6.0.0'}), + ('browser-base.json', {'files': []}), ('phenomena/audio/loop.json', {'packages': []}), + ('ui/audio/loop.json', {'packages': []})]: + write(assets / name, value) + return SimpleNamespace(root=root, manifest=manifest, game=game) + + +def test_prepare_and_selection_include_both_pointer_releases(publication, tmp_path): + (publication.root / 'releases/game-synthetic/untracked.mjs').write_text('// never release this') + output = tmp_path / 'output' + result = subprocess.run([sys.executable, str(TOOLS / 'prepare_deployment.py'), + '--publication', str(publication.root), '--output', str(output), '--provider', 's3', + '--object-prefix=', '--encoding', 'identity'], text=True, capture_output=True) + assert result.returncode == 0, result.stderr + filename = output / 'upload-manifest.json' + doc = json.loads(filename.read_bytes()) + assert doc['release'] == 'stage-synthetic' + assert doc['releases'] == ['stage-synthetic', 'game-synthetic'] + assert 'releases/game-synthetic/untracked.mjs' not in {row['path'] for row in doc['files']} + uploader = load('upload_oss') + def select(release): + args = SimpleNamespace(manifest=filename, scope='release', release_id=release, mutable_path=[], + allow_mutable_update=False, expect_previous_sha256=None, limit=None) + return uploader.load_selection(args, object_prefix='', schemas=('moly-object-upload/1',))[1] + assert {r['path'] for r in select('game-synthetic')} == { + 'releases/game-synthetic/game-boot.mjs', 'releases/game-synthetic/integrity.json'} + assert all(r['path'].startswith('releases/stage-synthetic/') for r in select('stage-synthetic')) + with pytest.raises(uploader.SafeFailure, match='explicit_release_id_must_match_manifest'): + select('unrelated') + doc['releases'][1] = 'unrelated' + write(filename, doc) + with pytest.raises(uploader.SafeFailure, match='release_pointers_do_not_match_manifest'): + select('unrelated') + + +@pytest.mark.parametrize('mutation', ['module', 'snapshot', 'coordinate', 'identity', 'missing', 'bytes', 'path']) +def test_changed_game_release_or_pointer_is_rejected(publication, mutation): + prepare = load('prepare_deployment') + root, game = publication.root, publication.game + if mutation == 'module': + game['release']['module'] = '/moly/releases/unrelated/game-boot.mjs' + elif mutation == 'snapshot': + game['snapshots'][0]['id'] = 'cn-unrelated' + elif mutation == 'coordinate': + game['release']['coordinateContract'] = 'unrelated' + elif mutation == 'identity': + file = root / 'releases/game-synthetic/integrity.json' + value = json.loads(file.read_bytes()); value['releaseId'] = 'unrelated'; write(file, value) + elif mutation == 'missing': + (root / 'releases/game-synthetic/game-boot.mjs').unlink() + elif mutation == 'bytes': + (root / 'releases/game-synthetic/game-boot.mjs').write_text('tampered') + elif mutation == 'path': + file = root / 'releases/game-synthetic/integrity.json' + value = json.loads(file.read_bytes()); value['hashes']['../outside'] = 'a' * 64; write(file, value) + write(root / 'game/manifest.json', game) + # Read the original manifest, since Python fixture nested dictionaries may share identity. + manifest = json.loads((root / 'manifest.json').read_bytes()) + with pytest.raises((ValueError, RuntimeError, OSError)): + prepare.publication_release_files(root, manifest, '/moly/') + + +def test_single_release_publication_remains_supported(publication): + (publication.root / 'game/manifest.json').unlink() + releases, files, count = load('prepare_deployment').publication_release_files( + publication.root, publication.manifest, '/moly/') + assert releases == ['stage-synthetic'] + assert count == 1 + assert set(files) == {'releases/stage-synthetic/embed.mjs', 'releases/stage-synthetic/integrity.json'} + + +def test_combined_legacy_engine_is_verified_once(publication): + root = publication.root + stage = root / 'releases/stage-synthetic' + body = (root / 'releases/game-synthetic/game-boot.mjs').read_bytes() + (stage / 'game-boot.mjs').write_bytes(body) + integrity = json.loads((stage / 'integrity.json').read_bytes()) + integrity['hashes']['game-boot.mjs'] = hashlib.sha256(body).hexdigest() + write(stage / 'integrity.json', integrity) + publication.game['release']['id'] = 'stage-synthetic' + publication.game['release']['module'] = '/moly/releases/stage-synthetic/game-boot.mjs' + write(root / 'game/manifest.json', publication.game) + releases, files, count = load('prepare_deployment').publication_release_files(root, publication.manifest, '/moly/') + assert releases == ['stage-synthetic'] + assert count == 2 + assert len(files) == 3 diff --git a/tools/deployment/prepare_deployment.py b/tools/deployment/prepare_deployment.py index ef4e5b0..8a0af66 100644 --- a/tools/deployment/prepare_deployment.py +++ b/tools/deployment/prepare_deployment.py @@ -298,6 +298,73 @@ def delivery_profile(mode, generate): return profile + +def publication_release_files(publication, manifest, url_prefix): + """Verify only releases selected by the stage manifest and game pointer.""" + identifier = re.compile(r'[a-z0-9][a-z0-9._-]{0,95}') + primary = manifest['release']['id'] + if not isinstance(primary, str) or not identifier.fullmatch(primary): + raise ValueError('Invalid primary release identity') + releases = [primary] + game_path = publication / 'game/manifest.json' + game = json.loads(game_path.read_bytes()) if game_path.is_file() else None + if game is not None: + record = game.get('release', {}) + game_id = record.get('id') + expected_snapshots = [{ + 'id': row['id'], 'region': row['region'], 'version': row['version'], + 'assets': row['assets'], 'packs': row.get('packs') is True, + 'assetCatalog': row.get('assetCatalog') if row.get('packs') is True else None, + } for row in manifest['snapshots']] + if (game.get('publisher') != 'moly-release-artifact-v1' or game.get('schemaVersion') != 1 + or game.get('prefix') != url_prefix + or not isinstance(game_id, str) or not identifier.fullmatch(game_id) + or record.get('module') != f'{url_prefix}releases/{game_id}/game-boot.mjs' + or record.get('abi') != 2 + or not record.get('coordinateContract') + or record['coordinateContract'] != manifest['release'].get('coordinateContract') + or game.get('snapshots') != expected_snapshots): + raise ValueError('Game pointer does not match the publication contract and snapshots') + if game_id not in releases: + releases.append(game_id) + files, integrity_count = {}, 0 + for release_id in releases: + root = publication / 'releases' / release_id + if root.is_symlink() or not root.resolve().is_relative_to(publication.resolve()): + raise ValueError('Release root escapes publication') + integrity_file = root / 'integrity.json' + if integrity_file.is_symlink(): + raise ValueError('Release integrity must be a regular local file') + integrity = json.loads(integrity_file.read_bytes()) + if ('releaseId' in integrity and integrity['releaseId'] != release_id + or 'contractVersion' in integrity and integrity['contractVersion'] != manifest.get('schemaVersion') + or 'coordinateContract' in integrity and integrity['coordinateContract'] != manifest['release'].get('coordinateContract')): + raise ValueError(f'Release integrity identity mismatch: {release_id}') + hashes = integrity.get('hashes') + if not isinstance(hashes, dict) or not hashes: + raise ValueError(f'Release has no integrity membership: {release_id}') + if game is not None and game['release']['id'] == release_id: + if (integrity.get('schemaVersion') != 1 or integrity.get('releaseId') != release_id + or integrity.get('contractVersion') != manifest.get('schemaVersion') + or integrity.get('coordinateContract') != game['release']['coordinateContract'] + or 'game-boot.mjs' not in hashes): + raise ValueError('Game release has no matching integrity contract') + for name, expected_hash in hashes.items(): + safe_relative(name) + if (not re.fullmatch(r'[a-f0-9]{64}', expected_hash or '') + or name == 'integrity.json' or name.endswith(('.gz', '.br'))): + raise ValueError(f'Invalid release integrity member: {name}') + source = root / name + if (not source.resolve().is_relative_to(root.resolve()) + or source.is_symlink() or any(parent.is_symlink() for parent in source.parents if parent != publication)): + raise ValueError(f'Release member escapes publication: {name}') + if digest(source.read_bytes()) != expected_hash: + raise RuntimeError(f'Release integrity mismatch: {release_id}/{name}') + files[f'releases/{release_id}/{name}'] = source + integrity_count += 1 + files[f'releases/{release_id}/integrity.json'] = integrity_file + return releases, files, integrity_count + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--publication', type=Path, required=True) @@ -354,16 +421,10 @@ def add(logical, source): # A publication with the game profile carries the game page's pointer. if (publication / 'game' / 'manifest.json').is_file(): add('game/manifest.json', publication / 'game' / 'manifest.json') - release_id = safe_relative(manifest['release']['id']) - release_root = publication / 'releases' / release_id - release_integrity = json.loads((release_root / 'integrity.json').read_bytes()) - for name, expected_hash in release_integrity['hashes'].items(): - safe_relative(name) - if digest((release_root / name).read_bytes()) != expected_hash: - raise RuntimeError(f'Release integrity mismatch: {name}') - for file in (publication / 'releases' / release_id).rglob('*'): - if file.is_file() and file.suffix not in {'.gz', '.br'}: - add(file.relative_to(publication).as_posix(), file) + release_ids, release_files, integrity_count = publication_release_files(publication, manifest, args.url_prefix) + release_id = release_ids[0] + for logical, source in release_files.items(): + add(logical, source) reports = {} for snapshot in manifest['snapshots']: snapshot_id = safe_relative(snapshot['id']) @@ -408,7 +469,7 @@ def add(logical, source): print(json.dumps({snapshot['region']: {k: report[k] for k in ['files', 'bytes', 'excludedBytes', 'missing', 'browserBaseMismatch']}, **({'engineReadsUnpublished': report['engineReads']['unpublished']} if 'engineReads' in report else {})}), flush=True) report_path = output / 'audit.json' - write_json(report_path, {'regions': reports, 'releaseIntegrityFilesChecked': len(release_integrity['hashes'])}) + write_json(report_path, {'regions': reports, 'releaseIntegrityFilesChecked': integrity_count}) if args.engine_reads and set(engine_logs) - set(reports): raise ValueError(f'--engine-reads names a region without a snapshot: {sorted(set(engine_logs) - set(reports))}') if any(row['missing'] or row['browserBaseMismatch'] or row.get('engineReads', {}).get('unpublished') @@ -456,7 +517,7 @@ def add(logical, source): for kind in ['identity', 'gzip', 'br']}} result = {'schema': 'moly-object-upload/1' if args.provider == 's3' else 'moly-oss-upload/1', 'deliveryMode': 'fixed', 'bucket': None, 'objectPrefix': prefix, 'urlPrefix': args.url_prefix, - 'release': release_id, 'snapshots': [s['id'] for s in manifest['snapshots']], + 'release': release_id, 'releases': release_ids, 'snapshots': [s['id'] for s in manifest['snapshots']], 'encodingPolicy': args.encoding, 'httpDelivery': profile, 'totals': totals, 'invalidSidecarsSkipped': invalid_sidecars, 'notes': ['No cloud writes performed.', 'Upload Content-Encoding exactly as listed; identity means omit that header.', @@ -468,7 +529,7 @@ def add(logical, source): 'Staging hardlinks must never be edited in place.'], 'files': entries} write_json(output / 'upload-manifest.json', result) write_json(report_path, {'regions': reports, 'totals': totals, 'invalidSidecarsSkipped': invalid_sidecars, - 'releaseIntegrityFilesChecked': len(release_integrity['hashes'])}) + 'releaseIntegrityFilesChecked': integrity_count}) print(json.dumps({'totals': totals, 'invalidSidecars': len(invalid_sidecars), 'output': str(output)}), flush=True) diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 7e5c9c5..02bbfd1 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -273,7 +273,37 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) translation_base = None if (not snapshots and translation is None and font_catalog is None and resource_catalog is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): raise SafeFailure('invalid_snapshot_identities') - if args.scope == 'release' and (args.release_id != doc.get('release') or not IDENTIFIER.fullmatch(args.release_id or '')): + releases = doc.get('releases', [doc['release']] if doc.get('release') else []) + if (not isinstance(releases, list) or len(releases) > 2 + or any(not isinstance(value, str) or not IDENTIFIER.fullmatch(value) for value in releases) + or len(set(releases)) != len(releases) + or (doc.get('release') is not None and doc['release'] not in releases) + or (releases and (resource_catalog is not None or font_catalog is not None or translation is not None))): + raise SafeFailure('invalid_release_identities') + if len(releases) > 1: + # A second release must be the actual game pointer target, not an + # arbitrary directory added to the upload inventory by an operator. + pointers = {} + for logical in ('manifest.json', 'game/manifest.json'): + rows = [row for row in doc.get('files', []) if row.get('path') == logical] + if len(rows) != 1: + raise SafeFailure('missing_release_pointer', path=logical) + try: + raw = Path(rows[0]['sourceRaw']).read_bytes() + if hashlib.sha256(raw).hexdigest() != rows[0]['decodedSha256']: + raise SafeFailure('local_descriptor_hash_mismatch', path=logical) + pointers[logical] = json.loads(raw) + except (OSError, KeyError, TypeError, ValueError) as error: + raise SafeFailure('cannot_read_release_pointer', path=logical) from None + primary = pointers['manifest.json'].get('release', {}).get('id') + game = pointers['game/manifest.json'] + game_id = game.get('release', {}).get('id') + if (primary != doc.get('release') or releases != list(dict.fromkeys([primary, game_id])) + or game.get('publisher') != 'moly-release-artifact-v1' or game.get('schemaVersion') != 1 + or game.get('prefix') != doc.get('urlPrefix') + or game.get('release', {}).get('module') != f"{doc.get('urlPrefix')}releases/{game_id}/game-boot.mjs"): + raise SafeFailure('release_pointers_do_not_match_manifest') + if args.scope == 'release' and args.release_id not in releases: raise SafeFailure('explicit_release_id_must_match_manifest') if args.scope == 'mutable': if not args.allow_mutable_update or len(args.mutable_path) != 1: diff --git a/tools/head-portraits/export.mjs b/tools/head-portraits/export.mjs index fca4a59..cbb4976 100644 --- a/tools/head-portraits/export.mjs +++ b/tools/head-portraits/export.mjs @@ -120,7 +120,7 @@ export async function exportHeads(options) { let page; const errors = []; try { - browser = await chromium.launch({ headless: true, ...(options.channel ? { channel: options.channel } : {}) }); + browser = await chromium.launch({ headless: !options.headed, ...(options.channel ? { channel: options.channel } : {}) }); const browserVersion = browser.version(); page = await browser.newPage({ viewport: { width: 1024, height: 900 }, deviceScaleFactor: 1 }); page.setDefaultTimeout(40000); @@ -191,10 +191,10 @@ export async function exportHeads(options) { async function main() { const { values } = parseArgs({ options: { source: { type: 'string' }, out: { type: 'string' }, region: { type: 'string' }, version: { type: 'string' }, - units: { type: 'string' }, channel: { type: 'string' }, python: { type: 'string' }, help: { type: 'boolean' }, + units: { type: 'string' }, headed: { type: 'boolean' }, channel: { type: 'string' }, python: { type: 'string' }, help: { type: 'boolean' }, } }); if (values.help) { - console.log('node tools/head-portraits/export.mjs --source --out --region [--version ] [--units 1,17] [--channel chrome] [--python python]'); + console.log('node tools/head-portraits/export.mjs --source --out --region [--version ] [--units 1,17] [--channel chrome] [--headed] [--python python]'); return; } for (const key of ['source', 'out', 'region']) if (!values[key]) throw new Error(`Missing --${key}`); From a6099c5c2dcdabf763915973ea157e93bff2259c Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:27:13 +0800 Subject: [PATCH 11/15] fix(publication): require the primary release identity --- tests/test_independent_releases.py | 5 +++++ tools/deployment/upload_oss.py | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/test_independent_releases.py b/tests/test_independent_releases.py index df51eaf..ac677cc 100644 --- a/tests/test_independent_releases.py +++ b/tests/test_independent_releases.py @@ -75,6 +75,11 @@ def select(release): assert all(r['path'].startswith('releases/stage-synthetic/') for r in select('stage-synthetic')) with pytest.raises(uploader.SafeFailure, match='explicit_release_id_must_match_manifest'): select('unrelated') + original_release = doc.pop('release') + write(filename, doc) + with pytest.raises(uploader.SafeFailure, match='invalid_release_identities'): + select('game-synthetic') + doc['release'] = original_release doc['releases'][1] = 'unrelated' write(filename, doc) with pytest.raises(uploader.SafeFailure, match='release_pointers_do_not_match_manifest'): diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 02bbfd1..9ef576d 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -277,6 +277,7 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) if (not isinstance(releases, list) or len(releases) > 2 or any(not isinstance(value, str) or not IDENTIFIER.fullmatch(value) for value in releases) or len(set(releases)) != len(releases) + or (releases and doc.get('release') is None) or (doc.get('release') is not None and doc['release'] not in releases) or (releases and (resource_catalog is not None or font_catalog is not None or translation is not None))): raise SafeFailure('invalid_release_identities') From 32c7d7649a8397c36ddd3b081c8dba40b36c4351 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:55:24 +0800 Subject: [PATCH 12/15] fix(audio): retain complete banks across weather exports --- src/phenomena/audio.py | 72 ++++++++++++++++++++++++++++++----- src/phenomena/environments.py | 2 +- tests/test_audio_preserve.py | 70 ++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 tests/test_audio_preserve.py diff --git a/src/phenomena/audio.py b/src/phenomena/audio.py index e188a9b..e9ccae9 100644 --- a/src/phenomena/audio.py +++ b/src/phenomena/audio.py @@ -33,6 +33,8 @@ the random-without-repeat type. """ import re +import hashlib +import json import shutil import struct import subprocess @@ -163,13 +165,60 @@ def attach_sequences(entry, data): class Library: """Audio of one run: archives always, waveforms when the decoder is present.""" - def __init__(self, directory, prefix, decoder=None, transcoder=None): + def __init__(self, directory, prefix, decoder=None, transcoder=None, *, preserve_existing=False): self.root = Path(directory) self.prefix = prefix self.decoder = tool(DECODER, decoder) self.transcoder = tool(TRANSCODER, transcoder) self.packages = [] self.unsupported = [] + self.previous = None + if preserve_existing and (self.root / "loop.json").is_file(): + self.previous = json.loads((self.root / "loop.json").read_bytes()) + # Capture identity before add() can write an archive at its old path. + for entry in self.previous.get("packages", []): + if not entry.get("sourceSha256"): + archive = self._existing_file(entry.get("archive")) + if archive is not None: + entry["sourceSha256"] = hashlib.sha256(archive.read_bytes()).hexdigest() + + def _existing_file(self, logical): + if not isinstance(logical, str) or not logical.startswith(self.prefix + "/"): + return None + target = self.root / logical[len(self.prefix) + 1:] + if not target.resolve().is_relative_to(self.root.resolve()): + raise ValueError("existing audio path escaped output root") + return target if target.is_file() else None + + def _preserved_packages(self): + if self.previous is None: + return self.packages + current = {(row["package"], Path(row["archive"]).name): row for row in self.packages} + combined = [] + for prior in self.previous.get("packages", []): + key = (prior["package"], Path(prior["archive"]).name) + fresh = current.pop(key, None) + if fresh is None: + combined.append(prior) + continue + if not prior.get("sourceSha256") or prior["sourceSha256"] != fresh["sourceSha256"]: + raise ValueError("audio archive identity changed; use a fresh output root") + # A weather subset cannot erase the whole bank the corpus decoded. + keys = {(row.get("cue"), row.get("subsong")) for row in fresh["streams"]} + retained = [row for row in prior.get("streams", []) + if (row.get("cue"), row.get("subsong")) not in keys] + if any(not any(self._existing_file(row.get(kind)) is not None + for kind in ("wav", "ogg")) for row in retained): + raise ValueError("existing audio stream is missing; repair the corpus before extraction") + merged = {**fresh, "streams": [*retained, *fresh["streams"]]} + for name in ("archiveSetRevision", "archiveSetCount", "archiveSetIndex"): + if name in prior: + merged[name] = prior[name] + archive = self._existing_file(fresh["archive"]) + attach_sequences(merged, archive.read_bytes()) + combined.append(merged) + combined.extend(current.values()) + return combined @property def status(self): @@ -211,9 +260,15 @@ def add(self, package, name, data, cues=None, *, namespace=None, report_missing= prefix = f"{self.prefix}/{relative.as_posix()}" directory.mkdir(parents=True, exist_ok=True) archive = directory / f"{name}.acb" + source_sha = hashlib.sha256(data).hexdigest() + for prior in (self.previous or {}).get("packages", []): + if prior["package"] == package and Path(prior["archive"]).name == archive.name: + if prior.get("sourceSha256") != source_sha: + raise ValueError("audio archive identity changed; use a fresh output root") archive.write_bytes(data) entry = {"package": package, "archive": f"{prefix}/{archive.name}", - "archiveBytes": len(data), "status": self.status, "streams": []} + "archiveBytes": len(data), "sourceSha256": source_sha, + "status": self.status, "streams": []} if not self.decoder: entry["error"] = NO_DECODER self.packages.append(entry) @@ -256,13 +311,10 @@ def add(self, package, name, data, cues=None, *, namespace=None, report_missing= def finish(self): """Write the loop sidecar and return the index-level audio document. - The write is a wholesale replacement, not a merge: the file on disk - becomes exactly this run's package list. Merging with what a previous - run wrote lives on the corpus side (``audio_corpus._merge_loop``), - which is also what the corpus's resume reads as its skip roster — so - re-running the phenomena job in place over a directory the corpus - already wrote replaces that roster whole, and the next corpus run - re-decodes every package the phenomena run did not carry. + By default the index becomes exactly this run's package list. Weather + callers opt into preserve_existing so re-exporting its cue subset does + not erase the rest of the same-source sound corpus. Overlapping banks + must retain their exact archive hash; changing releases needs a new root. """ # Only tool names, never the paths they were found at: this document ships # next to the audio and would otherwise carry one machine's directory layout. @@ -270,7 +322,7 @@ def finish(self): "decoderPresent": bool(self.decoder), "transcoder": TRANSCODER, "transcoderPresent": bool(self.transcoder), - "packages": self.packages} + "packages": self._preserved_packages()} if not self.decoder: document["error"] = NO_DECODER elif not self.transcoder: diff --git a/src/phenomena/environments.py b/src/phenomena/environments.py index 0096090..da4f036 100644 --- a/src/phenomena/environments.py +++ b/src/phenomena/environments.py @@ -995,7 +995,7 @@ def extract_phenomena(bundles, out_dir, bundle_root=None, master=None, "schemaVersion": collision_document["schemaVersion"], **collision_document["summary"]} - library = Library(out / "audio", "audio", vgmstream, ffmpeg) + library = Library(out / "audio", "audio", vgmstream, ffmpeg, preserve_existing=True) sound_packages = {} wanted_cues = sorted({str(row.get("cue")) for row in sound_rows if row.get("cue")}) diff --git a/tests/test_audio_preserve.py b/tests/test_audio_preserve.py new file mode 100644 index 0000000..32fd00b --- /dev/null +++ b/tests/test_audio_preserve.py @@ -0,0 +1,70 @@ +"""Weather extraction must not erase a complete audio corpus registry.""" +import hashlib +import json +from pathlib import Path + +import pytest + +from phenomena import audio + + +def source(tmp_path): + root = tmp_path / 'audio'; (root / 'se').mkdir(parents=True) + raw = b'synthetic same-source cue archive' + (root / 'se/se.acb').write_bytes(raw) + (root / 'se/walk.ogg').write_bytes(b'walk') + (root / 'se/wind.ogg').write_bytes(b'wind') + prior = {'package': 'shared', 'archive': 'audio/se/se.acb', 'status': 'succeeded', + 'archiveSetRevision': 2, 'archiveSetCount': 1, 'archiveSetIndex': 0, + 'streams': [{'cue': 'walk', 'subsong': 1, 'ogg': 'audio/se/walk.ogg'}, + {'cue': 'wind', 'subsong': 2, 'ogg': 'audio/se/wind.ogg'}]} + untouched = {'package': 'voice', 'archive': 'audio/voice/voice.acb', 'streams': []} + (root / 'loop.json').write_text(json.dumps({'packages': [prior, untouched]})) + return root, raw, prior, untouched + + +def fresh(raw): + return {'package': 'shared', 'archive': 'audio/se/se.acb', 'status': 'succeeded', + 'sourceSha256': hashlib.sha256(raw).hexdigest(), + 'streams': [{'cue': 'wind', 'subsong': 2, 'ogg': 'audio/se/wind.ogg'}]} + + +def test_weather_subset_preserves_other_cues_and_other_banks(tmp_path, monkeypatch): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + library.packages = [fresh(raw)] + monkeypatch.setattr(audio, 'attach_sequences', lambda row, data: row.update( + sequenceTest={'cues': [s['cue'] for s in row['streams']], 'source': hashlib.sha256(data).hexdigest()})) + result = library.finish() + assert result['packages'][1] == untouched + combined = result['packages'][0] + assert combined['streams'] == prior['streams'] + assert combined['archiveSetRevision'] == 2 + assert combined['sequenceTest']['cues'] == ['walk', 'wind'] + assert combined['sourceSha256'] == hashlib.sha256(raw).hexdigest() + + +def test_changed_archive_fails_before_overwriting_audio(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + previous = (root / 'loop.json').read_bytes() + with pytest.raises(ValueError, match='identity changed'): + library.add('shared', 'se', b'another release') + assert (root / 'se/se.acb').read_bytes() == raw + assert (root / 'loop.json').read_bytes() == previous + + +def test_missing_preserved_stream_requires_repair(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + library.packages = [fresh(raw)] + (root / 'se/walk.ogg').unlink() + with pytest.raises(ValueError, match='stream is missing'): + library.finish() + + +def test_default_library_still_replaces_explicit_whole_output(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio') + library.packages = [fresh(raw)] + assert library.finish()['packages'] == [fresh(raw)] From b8113ef84f014dd11df86ca23ed759b1049e839a Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:02:40 +0800 Subject: [PATCH 13/15] fix(publish): reuse verified immutable HTTP representations --- tests/test_verified_representations.py | 171 +++++++++++++++++++ tools/deployment/README.md | 22 +++ tools/deployment/prepare_deployment.py | 11 +- tools/deployment/prepare_resources.py | 15 +- tools/deployment/resource_publication.py | 9 +- tools/deployment/verified_representations.py | 147 ++++++++++++++++ 6 files changed, 367 insertions(+), 8 deletions(-) create mode 100644 tests/test_verified_representations.py create mode 100644 tools/deployment/verified_representations.py diff --git a/tests/test_verified_representations.py b/tests/test_verified_representations.py new file mode 100644 index 0000000..84b026e --- /dev/null +++ b/tests/test_verified_representations.py @@ -0,0 +1,171 @@ +"""An explicit immutable representation index preserves bytes and exact headers.""" +import copy +import gzip +import hashlib +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).parents[1] / 'src')) +sys.path.insert(0, str(Path(__file__).parents[1] / 'tools/deployment')) +from verified_representations import VerifiedRepresentations, validate_reused_row, SCHEMA + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def fixture(tmp_path, raw=b'original bytes ' * 1000, encoding='gzip'): + source = tmp_path / 'source.bin' + source.write_bytes(raw) + body = gzip.compress(raw, compresslevel=9, mtime=1680000000) if encoding == 'gzip' else raw + (tmp_path / 'origin-body.bin').write_bytes(body) + logical = f'asset-store/blobs/{sha(raw)[:2]}/{sha(raw)}.bin' + row = {'path': logical, 'objectKey': logical, 'body': 'origin-body.bin', + 'sha256': sha(body), 'bytes': len(body), 'decodedSha256': sha(raw), 'decodedBytes': len(raw), + 'headers': {'ContentLength': len(body), 'ContentType': 'application/octet-stream', + 'ContentEncoding': None if encoding == 'identity' else encoding, + 'CacheControl': 'public, max-age=31536000, immutable', + 'Metadata': {'moly-sha256': sha(body), 'moly-decoded-sha256': sha(raw), + 'moly-decoded-bytes': str(len(raw))}}} + index = tmp_path / 'index.json' + index.write_text(json.dumps({'schema': SCHEMA, 'files': [row]}), encoding='utf-8') + return source, index, row, body + + +@pytest.mark.parametrize('encoding', ['identity', 'gzip']) +def test_exact_origin_body_and_headers_win_over_fresh_compression(tmp_path, encoding): + source, index, evidence, body = fixture(tmp_path, encoding=encoding) + reuse = VerifiedRepresentations(index) + result = reuse.select(source, evidence['path'], evidence['objectKey'], 'application/octet-stream') + row = {'path': evidence['path'], 'objectKey': evidence['objectKey'], **result.representation.manifest_fields} + result.apply(row) + assert result.representation.body == body + assert row['cacheControl'] == evidence['headers']['CacheControl'] + assert row['sha256'] == sha(body) and not row['generatedRepresentation'] + assert row['contentEncoding'] == encoding + assert validate_reused_row(row, source, 'application/octet-stream').representation.body == body + assert reuse.audit()['reused'] == [evidence['path']] + assert not reuse.audit()['unused'] + + +@pytest.mark.parametrize('change', ['stored_hash', 'stored_length', 'decoded_hash', 'decoded_length', + 'header_length', 'header_metadata', 'unsafe_body', 'duplicate']) +def test_index_rejects_tampered_identity_and_headers(tmp_path, change): + source, index, row, body = fixture(tmp_path) + if change == 'stored_hash': row['sha256'] = 'a' * 64 + elif change == 'stored_length': row['bytes'] += 1 + elif change == 'decoded_hash': row['decodedSha256'] = 'a' * 64 + elif change == 'decoded_length': row['decodedBytes'] += 1 + elif change == 'header_length': row['headers']['ContentLength'] += 1 + elif change == 'header_metadata': row['headers']['Metadata']['moly-decoded-bytes'] = 'wrong' + elif change == 'unsafe_body': row['body'] = '../origin-body.bin' + rows = [row, copy.deepcopy(row)] if change == 'duplicate' else [row] + index.write_text(json.dumps({'schema': SCHEMA, 'files': rows}), encoding='utf-8') + with pytest.raises(ValueError): + VerifiedRepresentations(index).select(source, row['path'], row['objectKey'], 'application/octet-stream') + + +def test_actual_body_source_and_compression_are_checked_not_boolean_claims(tmp_path): + source, index, row, body = fixture(tmp_path) + (tmp_path / row['body']).write_bytes(body[:-1] + b'x') + with pytest.raises(ValueError, match='stored body'): + VerifiedRepresentations(index).select(source, row['path'], row['objectKey'], 'application/octet-stream') + (tmp_path / row['body']).write_bytes(body) + source.write_bytes(b'changed source') + with pytest.raises(ValueError, match='decoded source'): + VerifiedRepresentations(index).select(source, row['path'], row['objectKey'], 'application/octet-stream') + + +def test_request_key_and_media_type_cannot_alias_reuse(tmp_path): + source, index, row, _ = fixture(tmp_path) + reuse = VerifiedRepresentations(index) + with pytest.raises(ValueError, match='URL or media type'): + reuse.select(source, row['path'], 'another-prefix/' + row['objectKey'], 'application/octet-stream') + with pytest.raises(ValueError, match='URL or media type'): + reuse.select(source, row['path'], row['objectKey'], 'image/png') + assert VerifiedRepresentations().select(source, row['path'], row['objectKey'], 'application/octet-stream') is None + + +def test_upload_rechecks_index_hash_and_exact_selected_fields(tmp_path): + source, index, evidence, _ = fixture(tmp_path) + reused = VerifiedRepresentations(index).select(source, evidence['path'], evidence['objectKey'], 'application/octet-stream') + row = {'path': evidence['path'], 'objectKey': evidence['objectKey'], **reused.representation.manifest_fields} + reused.apply(row) + for field, value in [('cacheControl', 'public, max-age=60'), ('sha256', '0' * 64), ('bytes', 1)]: + altered = {**row, field: value} + with pytest.raises(ValueError): validate_reused_row(altered, source, 'application/octet-stream') + index.write_text(index.read_text(encoding='utf-8') + ' ', encoding='utf-8') + with pytest.raises(ValueError, match='index changed'): + validate_reused_row(row, source, 'application/octet-stream') + + +def test_resource_prepare_and_validator_preserve_existing_immutable_header(tmp_path, monkeypatch): + import prepare_resources + from pack.groups import build_groups + from resource_publication import validate_upload + source, store = tmp_path / 'root', tmp_path / 'store' + files = {'source.json': b'{"source":{"region":"jp","appVersion":"7.0.0"}}', + 'mysekai-fixtures.json': b'{"region":"jp","gameVersion":"7.0.0"}', + 'browser-base.json': b'{"files":[]}', 'phenomena/audio/loop.json': b'{"packages":[]}', + 'ui/audio/loop.json': b'{"packages":[]}', 'ui/test.json': b'{"test":"content"}'} + for name, body in files.items(): + path = source / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body) + build_groups(source, store, '7.0.0', region='jp') + selector = json.loads((store / 'channels/jp/stable.json').read_bytes()) + catalog = Path(selector['catalog']).stem + initial = prepare_resources.prepare(source, store, catalog, tmp_path / 'initial', encoding='identity', stage=True) + existing = next(row for row in initial['files'] if '/blobs/' in row['path']) + evidence_dir = tmp_path / 'evidence'; evidence_dir.mkdir() + _, index, evidence, body = fixture(evidence_dir, Path(existing['sourceRaw']).read_bytes(), 'identity') + evidence['headers']['ContentType'] = existing['contentType'] + index.write_text(json.dumps({'schema': SCHEMA, 'files': [evidence]}), encoding='utf-8') + reused = prepare_resources.prepare(source, store, catalog, tmp_path / 'reused', encoding='gzip', stage=True, verified_representations=index) + validate_upload(reused) + row = next(row for row in reused['files'] if row['path'] == existing['path']) + assert row['cacheControl'] == 'public, max-age=31536000, immutable' + assert Path(row['source']).read_bytes() == body + row['cacheControl'] += ', no-transform' + with pytest.raises(ValueError, match='headers differ'): + validate_upload(reused) + + +def test_deployment_preparation_reuses_addressed_bundle_and_exact_origin_headers(tmp_path, monkeypatch): + import prepare_deployment + publication, output = tmp_path / 'publication', tmp_path / 'output' + root = publication / 'snapshots/jp-test/assets' + release = publication / 'releases/stage-test' + release.mkdir(parents=True) + script = b'export const ready = true;\n' + (release / 'stage.mjs').write_bytes(script) + (release / 'integrity.json').write_text(json.dumps({'hashes': {'stage.mjs': sha(script)}})) + (publication / 'manifest.json').write_text(json.dumps({'release': {'id': 'stage-test'}, + 'snapshots': [{'id': 'jp-test', 'region': 'jp', 'version': '7.0.0'}]})) + (publication / 'cache-worker.mjs').write_text('// worker\n') + for name, body in {'mysekai-fixtures.json': b'{"region":"jp","gameVersion":"7.0.0"}', + 'browser-base.json': b'{"files":[]}', + 'phenomena/audio/loop.json': b'{"packages":[]}', + 'ui/audio/loop.json': b'{"packages":[]}'}.items(): + target = root / name; target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes(body) + evidence_dir = tmp_path / 'evidence'; evidence_dir.mkdir() + raw = b'{"sharedDetails":[]}' + b' ' * 2048 + _, index, evidence, body = fixture(evidence_dir, raw, 'gzip') + logical = f'catalog-store/{sha(raw)}.json' + evidence['path'] = evidence['objectKey'] = logical + target = publication / logical; target.parent.mkdir(); target.write_bytes(raw) + evidence['headers']['ContentType'] = prepare_deployment.content_type(logical, target) + index.write_text(json.dumps({'schema': SCHEMA, 'files': [evidence]})) + catalog = publication / 'snapshots/jp-test/catalog'; catalog.mkdir() + (catalog / 'index.json').write_text(json.dumps({'schemaVersion': 2, 'details': ['/moly/' + logical]})) + monkeypatch.setattr(sys, 'argv', ['prepare_deployment.py', '--publication', str(publication), + '--output', str(output), '--exporter', str(Path(__file__).parents[1]), '--provider', 's3', + '--object-prefix=', '--encoding', 'identity', '--stage', '--verified-representations', str(index)]) + prepare_deployment.main() + manifest = json.loads((output / 'upload-manifest.json').read_bytes()) + row = next(row for row in manifest['files'] if row['path'] == logical) + assert row['contentEncoding'] == 'gzip' + assert row['cacheControl'] == evidence['headers']['CacheControl'] + assert Path(row['staging']).read_bytes() == body + assert row['sha256'] == sha(body) diff --git a/tools/deployment/README.md b/tools/deployment/README.md index ae9b976..9797ab7 100644 --- a/tools/deployment/README.md +++ b/tools/deployment/README.md @@ -112,3 +112,25 @@ This reduces network bytes only. Browser HTTP decoding restores original hashes; the existing decoded CacheStorage and live CPU/GPU assets do not become smaller. The old `moly-deploy` viewer/R2 pipeline uses a different client-side `.glb.br` contract and is not used or modified by this direct-URL deployment path. + + +### Reuse an existing immutable HTTP representation + +`prepare_resources.py` and `prepare_deployment.py` accept +`--verified-representations INDEX.json`. The default still chooses fresh +lossless representations. Use this explicit index when an immutable origin URL +already holds an equivalent body with different compression or cache headers. +The index has schema `moly-verified-http-representations/1` and a `files` array. +Each row names `path`, `objectKey`, a relative local `body`, stored `sha256` and +`bytes`, decoded `decodedSha256` and `decodedBytes`, and exact origin `headers`: +`ContentLength`, `ContentType`, `ContentEncoding` (null means identity), +`CacheControl`, and `Metadata` with `moly-sha256`, `moly-decoded-sha256`, and +`moly-decoded-bytes`. + +Only content-addressed immutable resource URLs are eligible. Preparation checks +the actual stored body, bounded lossless decoding, source identity, key, media +type and headers. The audit lists reused and unused index entries. Resource +upload validation repeats these checks against the index SHA-256 recorded in +each reused row. Keep the index and bodies until upload finishes. This feature +does not contact the origin or change its objects; upload conflict checks still +require an exact match and reject a changed immutable representation. diff --git a/tools/deployment/prepare_deployment.py b/tools/deployment/prepare_deployment.py index 8a0af66..6741674 100644 --- a/tools/deployment/prepare_deployment.py +++ b/tools/deployment/prepare_deployment.py @@ -379,6 +379,7 @@ def main(): parser.add_argument('--stage', action='store_true', help='Create immutable hardlink staging grouped by content encoding') parser.add_argument('--encoding', choices=['identity', 'gzip', 'best'], default='best', help='Generate/select verified lossless HTTP representations; best allows Brotli') parser.add_argument('--existing-encodings-only', action='store_true', help='Audit/reuse existing sidecars without generating new representations') + parser.add_argument('--verified-representations', type=Path, help='Explicit local index of verified immutable origin bodies and headers') parser.add_argument('--engine-reads', action='append', default=[], metavar='REGION=FILE', help='Request or console log of a browser run; the audit fails on a requested path the membership lacks (engine_reads.py)') args = parser.parse_args() @@ -393,6 +394,8 @@ def main(): sys.path.insert(0, str(args.exporter.resolve() / 'src')) from pack.groups import runtime_paths from pack.http_delivery import choose_representation + from verified_representations import VerifiedRepresentations + reuse = VerifiedRepresentations(args.verified_representations) profile = delivery_profile(args.encoding, not args.existing_encodings_only) manifest = json.loads((publication / 'manifest.json').read_bytes()) source_overrides = {} @@ -479,7 +482,9 @@ def add(logical, source): entries, invalid_sidecars = [], [] # All preferred representations are validated against the exact raw bytes. for index, (logical, source) in enumerate(sorted(files.items())): - representation = choose_representation(source, logical, output / 'representations', + media_type = content_type(logical, source) + reused = reuse.select(source, logical, prefix + logical, media_type) + representation = reused.representation if reused else choose_representation(source, logical, output / 'representations', mode=args.encoding, generate=not args.existing_encodings_only) selected, encoding, body = representation.source, representation.encoding, representation.body invalid_sidecars.extend(representation.invalid_sidecars) @@ -490,6 +495,8 @@ def add(logical, source): } if encoding != 'identity' or logical.startswith('asset-store/'): entry['cacheControl'] += ', no-transform' + if reused: + reused.apply(entry) address = STORE_OBJECT.fullmatch(logical) if address and entry['decodedSha256'] != address[1]: raise RuntimeError(f'Content-addressed object does not match its name: {logical}') @@ -529,7 +536,7 @@ def add(logical, source): 'Staging hardlinks must never be edited in place.'], 'files': entries} write_json(output / 'upload-manifest.json', result) write_json(report_path, {'regions': reports, 'totals': totals, 'invalidSidecarsSkipped': invalid_sidecars, - 'releaseIntegrityFilesChecked': integrity_count}) + 'releaseIntegrityFilesChecked': integrity_count, 'verifiedRepresentations': reuse.audit()}) print(json.dumps({'totals': totals, 'invalidSidecars': len(invalid_sidecars), 'output': str(output)}), flush=True) diff --git a/tools/deployment/prepare_resources.py b/tools/deployment/prepare_resources.py index 5bc92a6..f412105 100644 --- a/tools/deployment/prepare_resources.py +++ b/tools/deployment/prepare_resources.py @@ -10,9 +10,10 @@ from pack.http_delivery import choose_representation from prepare_deployment import browser_membership, content_type, delivery_profile, write_json from resource_publication import catalog_closure, digest, CACHE_CONTROL +from verified_representations import VerifiedRepresentations -def prepare(source, store, catalog_id, output, *, object_prefix='', url_prefix='/moly/', encoding='best', stage=False): +def prepare(source, store, catalog_id, output, *, object_prefix='', url_prefix='/moly/', encoding='best', stage=False, verified_representations=None): source, store, output = Path(source).resolve(strict=True), Path(store).resolve(strict=True), Path(output).resolve() for left, right in [(source, store), (source, output), (store, output)]: if left.is_relative_to(right) or right.is_relative_to(left): @@ -32,15 +33,20 @@ def prepare(source, store, catalog_id, output, *, object_prefix='', url_prefix=' if len(data) != entries[name]['bytes'] or digest(data) != entries[name]['content_sha256']: raise ValueError('resource catalogue content differs from the supplied source: ' + name) profile = delivery_profile(encoding, True) + reuse = VerifiedRepresentations(verified_representations) rows, invalid_sidecars = [], [] for name, item in sorted(objects.items()): logical = 'asset-store/' + name - representation = choose_representation(item['source'], logical, output / 'representations', mode=encoding) + media_type = content_type(logical, item['source']) + reused = reuse.select(item['source'], logical, object_prefix + logical, media_type) + representation = reused.representation if reused else choose_representation(item['source'], logical, output / 'representations', mode=encoding) if representation.decoded_sha256 != item['sha256'] or representation.decoded_bytes != item['bytes']: raise ValueError('store changed while preparing HTTP representations') row = {'path': logical, 'objectKey': object_prefix + logical, 'urlPath': url_prefix + logical, 'contentType': content_type(logical, item['source']), 'cacheControl': CACHE_CONTROL, **representation.manifest_fields} + if reused: + reused.apply(row) invalid_sidecars.extend(representation.invalid_sidecars) if stage: target = output / 'staging' / representation.encoding / logical @@ -62,7 +68,7 @@ def prepare(source, store, catalog_id, output, *, object_prefix='', url_prefix=' 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows), 'decodedBytes': sum(row['decodedBytes'] for row in rows)}, 'deliveryProfile': profile} write_json(output / 'audit.json', {'resourceCatalog': identity, 'source': audit, - 'verifiedStore': verified, 'invalidSidecars': invalid_sidecars}) + 'verifiedStore': verified, 'invalidSidecars': invalid_sidecars, 'verifiedRepresentations': reuse.audit()}) write_json(output / 'upload-manifest.json', manifest) return manifest @@ -76,10 +82,11 @@ def main(): parser.add_argument('--object-prefix', default='') parser.add_argument('--url-prefix', default='/moly/') parser.add_argument('--encoding', choices=['identity', 'gzip', 'best'], default='best') + parser.add_argument('--verified-representations', type=Path, help='Explicit local index of verified immutable origin bodies and headers') parser.add_argument('--stage', action='store_true') args = parser.parse_args() result = prepare(args.source, args.store, args.catalog, args.output, object_prefix=args.object_prefix, - url_prefix=args.url_prefix, encoding=args.encoding, stage=args.stage) + url_prefix=args.url_prefix, encoding=args.encoding, stage=args.stage, verified_representations=args.verified_representations) import json print(json.dumps({'resourceCatalog': result['resourceCatalog'], 'totals': result['totals']})) diff --git a/tools/deployment/resource_publication.py b/tools/deployment/resource_publication.py index 29400b0..fadbf53 100644 --- a/tools/deployment/resource_publication.py +++ b/tools/deployment/resource_publication.py @@ -123,11 +123,16 @@ def validate_upload(doc): for name, expected in objects.items(): logical = 'asset-store/' + name row = rows[logical] + media_type = content_type(logical, expected['source']) + reused = None + if 'representationReuse' in row: + from verified_representations import validate_reused_row + reused = validate_reused_row(row, expected['source'], media_type) if (Path(row['sourceRaw']).absolute() != expected['source'] or row['decodedSha256'] != expected['sha256'] or row['decodedBytes'] != expected['bytes'] - or row['bytes'] > expected['bytes'] + or (reused is None and row['bytes'] > expected['bytes']) or row['contentType'] != content_type(logical, expected['source']) - or row['cacheControl'] != CACHE_CONTROL or row.get('urlPath') != url_prefix + logical): + or (reused is None and row['cacheControl'] != CACHE_CONTROL) or row.get('urlPath') != url_prefix + logical): raise ValueError('resource object metadata differs from its catalogue') for alias in [row.get('source'), *([row['staging']] if 'staging' in row else [])]: if not isinstance(alias, str) or not alias: diff --git a/tools/deployment/verified_representations.py b/tools/deployment/verified_representations.py new file mode 100644 index 0000000..3d13871 --- /dev/null +++ b/tools/deployment/verified_representations.py @@ -0,0 +1,147 @@ +"""Reuse a previously observed immutable HTTP body without changing its URL. + +The explicit index is local evidence, never a cloud write permission. Every use +rechecks stored bytes, lossless decoding, addressed identity, and exact headers. +The uploader still independently compares the selected representation to origin. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path, PurePosixPath +import re + +from pack.http_delivery import Representation +from pack.verify import _decode_content + +SCHEMA = 'moly-verified-http-representations/1' +ADDRESS = re.compile(r'(?:asset-store/(?:blobs/[0-9a-f]{2}|packages|catalogs)|catalog-store)/([0-9a-f]{64})\.[a-z0-9]+') +HASH = re.compile(r'[0-9a-f]{64}') +CACHES = frozenset({'public, max-age=31536000, immutable', + 'public, max-age=31536000, immutable, no-transform'}) + + +def digest(body): + return hashlib.sha256(body).hexdigest() + + +def relative(name): + if (not isinstance(name, str) or not name or name.startswith('/') + or any(char in name for char in '\\:%?#') + or any(part in {'', '.', '..'} for part in name.split('/')) + or any(ord(char) < 32 or ord(char) == 127 for char in name)): + raise ValueError('unsafe representation evidence path') + return PurePosixPath(name) + + +@dataclass(frozen=True) +class ReusedRepresentation: + representation: Representation + headers: dict + proof: dict + + def apply(self, row): + # These are precisely the headers recorded at origin, not freshly + # chosen compression/cache policy. Identity means omit Content-Encoding. + row.update(contentType=self.headers['ContentType'], + cacheControl=self.headers['CacheControl'], + representationReuse=self.proof) + + +class VerifiedRepresentations: + def __init__(self, index=None, *, expected_sha256=None): + self.index = None + self.sha256 = None + self.rows = {} + self.used = set() + if index is None: + return + self.index = Path(index).resolve(strict=True) + raw = self.index.read_bytes() + self.sha256 = digest(raw) + if expected_sha256 is not None and self.sha256 != expected_sha256: + raise ValueError('representation index changed') + document = json.loads(raw) + if set(document) != {'schema', 'files'} or document['schema'] != SCHEMA or not isinstance(document['files'], list): + raise ValueError('invalid verified representation index') + for row in document['files']: + if not isinstance(row, dict) or set(row) != {'path', 'objectKey', 'body', 'sha256', 'bytes', 'decodedSha256', 'decodedBytes', 'headers'}: + raise ValueError('invalid verified representation row') + logical = row['path'] + address = ADDRESS.fullmatch(logical) if isinstance(logical, str) else None + if not address or address[1] != row['decodedSha256'] or logical in self.rows: + raise ValueError('representation identity is not a unique addressed object') + relative(logical) + relative(row['objectKey']) + relative(row['body']) + if any(not isinstance(row[key], str) or not HASH.fullmatch(row[key]) for key in ('sha256', 'decodedSha256')): + raise ValueError('invalid representation hash') + if any(type(row[key]) is not int or row[key] < 0 for key in ('bytes', 'decodedBytes')): + raise ValueError('invalid representation length') + headers = row['headers'] + if not isinstance(headers, dict) or set(headers) != {'ContentLength', 'ContentType', 'ContentEncoding', 'CacheControl', 'Metadata'}: + raise ValueError('invalid representation headers') + if (type(headers['ContentLength']) is not int or headers['ContentLength'] != row['bytes'] + or headers['ContentEncoding'] not in {None, 'identity', 'gzip', 'br'} + or headers['CacheControl'] not in CACHES + or not isinstance(headers['ContentType'], str) or not headers['ContentType'] + or any(ord(char) < 32 or ord(char) == 127 for char in headers['ContentType']) + or headers['Metadata'] != {'moly-sha256': row['sha256'], + 'moly-decoded-sha256': row['decodedSha256'], 'moly-decoded-bytes': str(row['decodedBytes'])}): + raise ValueError('representation headers disagree with verified identity') + self.rows[logical] = row + + def select(self, source, logical, object_key, content_type): + row = self.rows.get(logical) + if row is None: + return None + if row['objectKey'] != object_key or row['headers']['ContentType'] != content_type: + raise ValueError('representation URL or media type changed') + source = Path(source).resolve(strict=True) + raw = source.read_bytes() + if len(raw) != row['decodedBytes'] or digest(raw) != row['decodedSha256']: + raise ValueError('representation decoded source changed') + body_path = self.index.parent / relative(row['body']) + # Reject links before resolving, including an intermediate directory. + if body_path.is_symlink() or any(parent.is_symlink() for parent in body_path.parents if parent != self.index.parent): + raise ValueError('representation body must not use symlinks') + body_path = body_path.resolve(strict=True) + if not body_path.is_relative_to(self.index.parent) or not body_path.is_file(): + raise ValueError('representation body escapes evidence directory') + with body_path.open('rb') as stream: + body = stream.read(row['bytes'] + 1) + if len(body) != row['bytes'] or digest(body) != row['sha256']: + raise ValueError('representation stored body changed') + encoding = row['headers']['ContentEncoding'] or 'identity' + decoded = _decode_content(body, 'gzip' if encoding == 'gzip' else 'identity', + 'br' if encoding == 'br' else 'identity', row['decodedBytes']) + if len(decoded) != row['decodedBytes'] or digest(decoded) != row['decodedSha256'] or decoded != raw: + raise ValueError('representation does not decode to its addressed source') + self.used.add(logical) + representation = Representation(body_path, source, encoding, body, len(raw), digest(raw), + (), False, 'verified-immutable-origin') + return ReusedRepresentation(representation, row['headers'], + {'index': str(self.index), 'sha256': self.sha256}) + + def audit(self): + return {'index': None if self.index is None else str(self.index), 'sha256': self.sha256, + 'reused': sorted(self.used), 'unused': sorted(self.rows.keys() - self.used)} + + +def validate_reused_row(row, source, content_type): + """Revalidate reuse evidence immediately before an upload is selected.""" + proof = row.get('representationReuse') + if not isinstance(proof, dict) or set(proof) != {'index', 'sha256'} or not isinstance(proof['sha256'], str) or not HASH.fullmatch(proof['sha256']): + raise ValueError('invalid representation reuse proof') + index = VerifiedRepresentations(proof['index'], expected_sha256=proof['sha256']) + reused = index.select(source, row['path'], row['objectKey'], content_type) + if reused is None: + raise ValueError('representation reuse proof omits object') + expected = reused.representation.manifest_fields + for field in ('sha256', 'bytes', 'decodedSha256', 'decodedBytes', 'contentEncoding', 'deliveryMode', 'requiredAcceptEncoding'): + if row.get(field) != expected[field]: + raise ValueError('representation manifest differs from verified evidence') + if row['cacheControl'] != reused.headers['CacheControl'] or row['contentType'] != reused.headers['ContentType']: + raise ValueError('representation manifest headers differ from verified evidence') + return reused From b4047cdae4c23f2a1d33edb712157465594a6d60 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:35:09 +0800 Subject: [PATCH 14/15] feat(harvest): export bulk hammer timeline and source UI --- src/fixtures/particles.py | 9 +- src/sites/bulk_harvest.py | 212 ++++++++++++++++++++++++++++++++ src/sites/step_items.py | 9 +- src/ui/action_icon.py | 4 +- src/ui/jp_decoders.py | 89 ++++++++++++++ src/ui/prefab_layout.py | 5 +- tests/test_bulk_harvest.py | 75 +++++++++++ tests/test_fixture_particles.py | 18 +++ tests/test_jp_decoders.py | 30 ++++- 9 files changed, 441 insertions(+), 10 deletions(-) create mode 100644 src/sites/bulk_harvest.py create mode 100644 tests/test_bulk_harvest.py diff --git a/src/fixtures/particles.py b/src/fixtures/particles.py index 55ac242..434b437 100644 --- a/src/fixtures/particles.py +++ b/src/fixtures/particles.py @@ -767,7 +767,8 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, domain the router gives them -- one package of a domain whose other packages are not read this way (the travel cannon of the site domain). A named package the store does not hold is not extracted; its document's - ``domain`` is still the one the router reports. + ``domain`` is the routed domain, or ``explicit`` when the router does not + claim that package. """ domains = tuple(domains) packages = tuple(sorted(set(packages))) @@ -786,7 +787,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, target_names, lookup_names = [], [] for name in sorted(names): target = route(name) - if target is not None and (target.domain in domains or name in packages): + if name in packages or target is not None and target.domain in domains: target_names.append(name) else: lookup_names.append(name) @@ -797,7 +798,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, documents = {} per_domain = {domain: 0 for domain in domains} for name in target_names: - per_domain.setdefault(route(name).domain, 0) + per_domain.setdefault(route(name).domain if route(name) else "explicit", 0) totals = {"packages": len(target_names), "lookupPackages": len(lookup_names), "emitters": 0, "materials": 0, "textures": 0, "unsupported": 0, "missing": 0, @@ -805,7 +806,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, "systemReadFailures": 0, "rendererReadFailures": 0} for name in target_names: routed = route(name) - document, file_name = _export_package(store, name, out, routed.domain, + document, file_name = _export_package(store, name, out, routed.domain if routed else "explicit", lambda directory: _Images(directory), source.materials, source.geometry) for emitter in document["emitters"]: diff --git a/src/sites/bulk_harvest.py b/src/sites/bulk_harvest.py new file mode 100644 index 0000000..7ab2efa --- /dev/null +++ b/src/sites/bulk_harvest.py @@ -0,0 +1,212 @@ +"""BulkHarvestToolTimelineView's runtime-bound hammer and camera streams. + +The timeline names its clips by serialized identity. The tool and field camera +are supplied by the view at runtime, so they cannot be inferred from null scene +bindings. Keep that contract explicit, and never assign a source object identity +to the exported camera proxy. +""" +from __future__ import annotations + +import hashlib +import json +import struct +from pathlib import Path +from types import SimpleNamespace + +from core.atomic import PUBLIC_FILE_MODE, write_json +from core.gltf import GLB +from core.jsonio import normalize +from fixtures.animations import decode_clip, embed, _write_channels +from . import step_items as steps +from .scenes import PackageExtract + +PACKAGE = "mysekai__player__motion__bulk_harvest" +PREFAB = "tl_bulk_harvest_main" +VIEW = "BulkHarvestToolTimelineView" +TOOL_PACKAGE = "mysekai__tool__mdl_site_tool_hammer01" +TOOL_PREFAB = "mdl_site_tool_hammer01" +CAMERA_NODE = "moly_bulk_camera" + + +def selected_tracks(tracks, targets, director, package): + """Resolve the three explicitly runtime-bound animation streams.""" + timeline = steps._timeline(tracks, director) + selected = {} + bindings = {steps._asset(row.get("key")): row.get("value") + for row in director.get("sceneBindings", [])} + for track in steps._walk_tracks(timeline.get("tracks")): + if track.get("class") != "AnimationTrack": + continue + name = track.get("name") + if name not in ("CharacterAnimator", "Fixture1", "camera") or name in selected: + raise ValueError(f"unsupported bulk animation stream: {name!r}") + identity = steps._asset(track.get("asset")) + if steps._asset(bindings.get(identity)) is not None: + raise ValueError(f"bulk runtime stream {name!r} unexpectedly has a scene binding") + clips = [] + for row in sorted((row for row in targets.get("keyedClips", []) + if steps._asset(row.get("track")) == identity), + key=lambda row: row.get("clipIndex", 0)): + target = row.get("target") or {} + source = steps._asset(target.get("sourceClip")) + if source is None or not target.get("clipName") or not target.get("targetPackage"): + raise ValueError(f"bulk stream {name!r} has an unresolved clip") + clips.append({"name": target["clipName"], "sourceClip": { + "file": source[0], "pathId": source[1]}, + "sourcePackage": target["targetPackage"]}) + if not clips: + raise ValueError(f"bulk stream {name!r} has no clips") + selected[name] = clips + if set(selected) != {"CharacterAnimator", "Fixture1", "camera"}: + raise ValueError("bulk timeline is missing a runtime animation stream") + return selected + + +def clip_records(store, clips): + files = {} + for clip in clips: + package = store.package(clip["sourcePackage"]) + if package is None: + raise ValueError(f"missing clip package: {clip['sourcePackage']}") + records = steps._Records(store, package) + found = records.lookup(steps._asset(clip["sourceClip"])) + if found is None or found[0].kinds[found[1]] != "AnimationClip": + raise ValueError(f"clip source is not an AnimationClip: {clip['sourceClip']}") + record, pid = found + if str(record.tree(pid).get("m_Name")) != clip["name"]: + raise ValueError(f"clip name disagrees with source: {clip['sourceClip']}") + files[record.archive] = record + return SimpleNamespace(lookup=lambda identity: ( + (files[identity[0]], int(identity[1])) + if identity[0] in files and int(identity[1]) in files[identity[0]].kinds else None)) + + +def _load_glb(path): + data = path.read_bytes() + if data[:4] != b"glTF" or struct.unpack_from(" dict: DECODERS_681 = {**SHARED, **ONLY_681} # The 7.0.0 layout: the four dialogs end at their click event, and the five # classes the 7.0.0 declaration changed read their added fields. + + +def decode_bulk_checkbox_control(r): + return {"checkObj": r.pptr(), "check": r.bool4(), "toggle": r.pptr()} + + +def decode_harvest_player_hud_700(r): + return editor_layout.refs(r, ("_staminaView", "_bulkHarvestChargeGaugeView", "_canvasGroup")) + + +def decode_stock_scroll_rect(r): + fields = {"m_Content": r.pptr(), "m_Horizontal": r.bool4(), "m_Vertical": r.bool4(), + "m_MovementType": r.i32(), "m_Elasticity": r.f32(), "m_Inertia": r.bool4(), + "m_DecelerationRate": r.f32(), "m_ScrollSensitivity": r.f32()} + fields.update(editor_layout.refs(r, ("m_Viewport", "m_HorizontalScrollbar", "m_VerticalScrollbar"))) + fields.update(m_HorizontalScrollbarVisibility=r.i32(), m_VerticalScrollbarVisibility=r.i32(), + m_HorizontalScrollbarSpacing=r.f32(), m_VerticalScrollbarSpacing=r.f32(), + m_OnValueChanged=talk.decode_unity_event(r)) + return fields + +def decode_bulk_harvest_button(r): + fields = decode_custom_button(r) + fields.update(editor_layout.refs(r, ("_cannotButton", "_cannotBalloon", "_iconLoader"))) + return fields + + +def decode_bulk_harvest_gauge(r): + return editor_layout.refs(r, ("_gaugeImage", "_canvasGroup", "_maxEffectRoot")) + + +def decode_bulk_harvest_settings(r): + from ui.window_animation import decode_common2_button_dialog + fields = decode_common2_button_dialog(r) + fields.update(editor_layout.refs(r, ("_siteTabGroup", "_checkBoxListViewPrefab", "_checkBoxListViewParent"))) + return fields + + +def decode_bulk_harvest_list(r): + return editor_layout.refs(r, ("_allCheckBox", "_groupViewRoot", "_groupViewBase")) + + +def decode_bulk_harvest_group(r): + return editor_layout.refs(r, ("_groupNameText", "_allCheckBox", "_checkBoxRoot", "_checkBoxBase")) + + +def decode_bulk_harvest_checkbox(r): + return editor_layout.refs(r, ("_checkBox", "_text")) + + +def decode_bulk_harvest_reward(r): + from ui.window_animation import decode_common1_button_dialog + fields = decode_common1_button_dialog(r) + fields.update(editor_layout.refs(r, ("_listView", "_shortageTextRoot", "_staminaShortageText", "_toolShortageText"))) + return fields + + +def decode_harvest_layer_700(r): + fields = {"layerCamera": r.pptr()} + fields.update(editor_layout.refs(r, ("_menuUIContent", "_selectedToolView", "_toolSelectorViewController", + "_harvestActionButton", "_toolNotHaveBalloon", "_harvestActionCannotButton", "_dashButton", "_bulkHarvestActionButton"))) + return fields + ONLY_700 = { + "Sekai.UIPartsThumbnailRewardListCell": lambda r: {"sizeX": r.f32(), "sizeY": r.f32(), "thumbnailRoots": talk.decode_pptr_list(r), "nameText": r.pptr()}, + "Sekai.UIPartsCheckBox": decode_bulk_checkbox_control, + "Sekai.Mysekai.HarvestPlayerHeadUpDisplay": decode_harvest_player_hud_700, + "UnityEngine.UI.ScrollRect": decode_stock_scroll_rect, + "Sekai.Mysekai.BulkHarvestActionButton": decode_bulk_harvest_button, + "Sekai.Mysekai.BulkHarvestChargeGaugeView": decode_bulk_harvest_gauge, + "Sekai.Mysekai.BulkHarvestCheckBoxListView": decode_bulk_harvest_list, + "Sekai.Mysekai.BulkHarvestGroupCheckBoxView": decode_bulk_harvest_group, + "Sekai.Mysekai.BulkHarvestTargetSettingCheckBox": decode_bulk_harvest_checkbox, + "Sekai.Mysekai.BulkHarvestTargetSettingDialog": decode_bulk_harvest_settings, + "Sekai.Mysekai.MysekaiBulkHarvestRewardDialog": decode_bulk_harvest_reward, + "Sekai.Mysekai.ScreenLayerMysekaiHarvest": decode_harvest_layer_700, + **photo_layout.DECODERS, "Sekai.UI.CustomScrollRect": decode_custom_scroll_rect_700, "Sekai.Mysekai.MysekaiMissionStartDialog": mission_layout.decode_mission_start_dialog, @@ -562,6 +637,20 @@ def decode_list_view_700(r: talk.Reader) -> dict: # (class name, serialized type hash hex) -> the layout the bytes carry. TYPE_LAYOUTS = { + ("Sekai.UIPartsThumbnailRewardListCell", "d2817fbfe87a2ef02b700815f2918477"): "7.0.0", + ("Sekai.Mysekai.HarvestPlayerHeadUpDisplay", "b0aefa4373b8001001f4e51227165a4e"): "7.0.0", + ("Sekai.UIPartsCheckBox", "beb6fd08a47ed26ebf63cc74f518fb08"): "7.0.0", + ("UnityEngine.UI.ScrollRect", "d5cd7f207e91ca8336c65a577a6fb2a0"): "7.0.0", + + ("Sekai.Mysekai.BulkHarvestActionButton", "0502d10b158ddd2b9dd787bb9dd58fb4"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestChargeGaugeView", "2167f64c1ddff3c0aece99da045ee93c"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestCheckBoxListView", "d2835e1e2684fe21d4e3bddcd2841cf3"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestGroupCheckBoxView", "b3511010ceb423f8f82c90a70d17ce65"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestTargetSettingCheckBox", "5ad4d5bf1f146309431caa4cfd96f525"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestTargetSettingDialog", "1faddd3b4959d3e99ca1afa0d2c7ac43"): "7.0.0", + ("Sekai.Mysekai.MysekaiBulkHarvestRewardDialog", "7231bc0e697f6ce50d2b7ce254736217"): "7.0.0", + ("Sekai.Mysekai.ScreenLayerMysekaiHarvest", "4dda12598a43e512cb119ab96403b0a9"): "7.0.0", + ("Sekai.UIPartsPhotoFrameThumbnail", "701a1b9241173c653af93358697fbf4a"): "7.0.0", ("Sekai.Mysekai.MysekaiSelectFrameView", "56a5828cb66f421adde139868f381dc6"): "7.0.0", ("Sekai.UIPartsPhotoFramePreview", "8d1a4f81285d407da7fc2608b914a06e"): "7.0.0", diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index 13a66eb..4b6288f 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -276,6 +276,7 @@ # The JP prefab registry: the product families, the referenced-only prefabs # and the census-only ones. JP_FAMILIES = { + "bulkharvest": ("BulkHarvestChargeGauge", "UIPartsThumbnailRewardListCell"), **SCREENS, **REFERENCED_FAMILIES, **{family: tuple(n for n in names if n not in _PRODUCT_PREFABS) @@ -290,6 +291,8 @@ # cell instance, adds a fourth item, the node's path under the page, and builds # the prefab from the page's own nodes (see follow_page_prefab_reference). POINTER_PREFABS = { + "BulkHarvestChargeGauge": ("ScreenLayerMysekaiHUD", "Sekai.Mysekai.HarvestPlayerHeadUpDisplay", "_bulkHarvestChargeGaugeView"), + "UIPartsThumbnailRewardListCell": ("MysekaiBulkHarvestRewardDialog", "Sekai.UI.ListView", "listViewItemPrefab"), "SiteEnvironmentSelectCell": ( "SiteEnvironmentContentList", "Sekai.UI.ListView", "listViewItemPrefab"), **CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS, @@ -3227,7 +3230,7 @@ def resource_key(prefab): reference, (go_pid, transform_pid, _) = follow_prefab_reference( env, holder_objects, holder_roots[0][1], item["via"][1], item["via"][2], Resolver({}, mono_index, None, env=env, decoders=decoders), - lambda name: _file_objects(env, name)) + lambda name: _file_objects(env, name), allow_nested=item["prefab"] == "BulkHarvestChargeGauge") if "targetSerializedFile" not in reference: reference["targetSerializedFile"] = holder_file item["target"] = (reference["targetSerializedFile"], go_pid) diff --git a/tests/test_bulk_harvest.py b/tests/test_bulk_harvest.py new file mode 100644 index 0000000..4abecaf --- /dev/null +++ b/tests/test_bulk_harvest.py @@ -0,0 +1,75 @@ +"""Runtime bindings must remain explicit and source-qualified.""" +from types import SimpleNamespace + +import pytest + +from core.gltf import GLB +from sites import bulk_harvest as bulk +from test_fixture_animations import _binding, _clip_tree + + +def _identity(pid): + return {"file": "CAB-synthetic", "pathId": str(pid)} + + +def _tables(): + names = ["CharacterAnimator", "Fixture1", "camera"] + tracks = [{"class": "AnimationTrack", "name": name, "asset": _identity(i)} + for i, name in enumerate(names, 1)] + source = {"timelines": [{"asset": _identity(9), "tracks": tracks}]} + targets = {"keyedClips": [{"track": t["asset"], "clipIndex": 0, + "target": {"sourceClip": _identity(20+i), "clipName": t["name"]+"_clip", + "targetPackage": "external" if i == 1 else bulk.PACKAGE}} + for i, t in enumerate(tracks)]} + director = {"timeline": _identity(9), "sceneBindings": [ + {"key": t["asset"], "value": None} for t in tracks]} + return source, targets, director + + +def test_bulk_streams_keep_the_declared_clip_package(): + source, targets, director = _tables() + selected = bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + assert selected["Fixture1"][0]["sourcePackage"] == "external" + assert selected["Fixture1"][0]["sourceClip"] == _identity(21) + director["sceneBindings"][1]["value"] = _identity(80) + with pytest.raises(ValueError, match="scene binding"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + + +def test_bulk_streams_refuse_unmodelled_or_missing_bindings(): + source, targets, director = _tables() + source["timelines"][0]["tracks"][2]["name"] = "second-camera" + with pytest.raises(ValueError, match="unsupported bulk"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + source, targets, director = _tables() + targets["keyedClips"][1]["target"].pop("sourceClip") + with pytest.raises(ValueError, match="unresolved clip"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + + +def test_camera_proxy_is_separate_and_has_no_source_object_identity(): + glb = GLB() + glb.g["nodes"] = [{"name": "source-root"}] + glb.g["scenes"][0]["nodes"] = [0] + tree = _clip_tree("camera", [_binding("", 1)], constant=[2.0, 3.0, 4.0]) + tree["m_MuscleClip"]["m_StopTime"] = 1.5 + records = SimpleNamespace(lookup=lambda _: (SimpleNamespace(tree=lambda _: tree), 1)) + clips = [{"name": "camera", "sourceClip": _identity(1)}] + bulk.append_camera(glb, records, clips) + assert glb.g["scenes"][0]["nodes"] == [0, 1] + assert glb.g["nodes"][1]["name"] == bulk.CAMERA_NODE + extras = glb.g["nodes"][1]["extras"] + assert "sourceObject" not in extras + assert extras["exportedProxy"]["cameraForwardCorrection"] == "runtime-y-pi" + assert clips[0]["sourceStopTime"] == 1.5 + assert glb.g["animations"][0]["channels"][0]["target"]["node"] == 1 + with pytest.raises(ValueError, match="already exists"): + bulk.append_camera(glb, records, clips) + + +def test_camera_rejects_a_non_root_curve(): + glb = GLB() + tree = _clip_tree("camera", [_binding("unbound", 1)], constant=[0.0, 0.0, 0.0]) + records = SimpleNamespace(lookup=lambda _: (SimpleNamespace(tree=lambda _: tree), 1)) + with pytest.raises(ValueError, match="unsupported bindings"): + bulk.append_camera(glb, records, [{"name": "camera", "sourceClip": _identity(1)}]) diff --git a/tests/test_fixture_particles.py b/tests/test_fixture_particles.py index 00b446e..00331a3 100644 --- a/tests/test_fixture_particles.py +++ b/tests/test_fixture_particles.py @@ -224,6 +224,24 @@ def test_emitter_and_renderer_pair_into_one_entry(tmp_path, monkeypatch): assert meta["renderers"] == 1 +def test_explicit_unrouted_package_is_extracted(tmp_path, monkeypatch): + name = "mysekai__player__motion__bulk_harvest" + package = _Package(name) + goid, _ = package.node("bulk") + package.particle_system(goid) + package.renderer(goid, []) + (tmp_path / name).touch() + monkeypatch.setattr(packages_module.UnityPy, "load", lambda _: package.finish()) + store = PackageStore([str(tmp_path / name)]) + out = tmp_path / "out" + extract_from_store(store, out, domains=(), packages=(name,)) + document = _document(out, name) + assert document["domain"] == "explicit" + assert len(document["emitters"]) == 1 + index = json.loads((out / "index.json").read_text()) + assert index["summary"]["perDomain"]["explicit"] == 1 + + def test_two_systems_share_the_node_renderer(tmp_path, monkeypatch): """Red when a node's several systems are fused into one emitter: the jacuzzi holds 103 systems on far fewer nodes, and each system keeps its own diff --git a/tests/test_jp_decoders.py b/tests/test_jp_decoders.py index a16b6b7..a8ee4fa 100644 --- a/tests/test_jp_decoders.py +++ b/tests/test_jp_decoders.py @@ -156,6 +156,15 @@ def test_700_chain_over_reads_681_bytes(name): "Sekai.UI.CustomScrollRect": "decode_custom_scroll_rect_700", } +BULK_CLASSES = { + "Sekai.UIPartsThumbnailRewardListCell", "Sekai.UIPartsCheckBox", + "Sekai.Mysekai.HarvestPlayerHeadUpDisplay", "UnityEngine.UI.ScrollRect", + "Sekai.Mysekai.BulkHarvestActionButton", "Sekai.Mysekai.BulkHarvestChargeGaugeView", + "Sekai.Mysekai.BulkHarvestCheckBoxListView", "Sekai.Mysekai.BulkHarvestGroupCheckBoxView", + "Sekai.Mysekai.BulkHarvestTargetSettingCheckBox", "Sekai.Mysekai.BulkHarvestTargetSettingDialog", + "Sekai.Mysekai.MysekaiBulkHarvestRewardDialog", "Sekai.Mysekai.ScreenLayerMysekaiHarvest", +} + def test_700_table_holds_the_changed_classes_only(): """The 7.0.0 table differs from the 6.8.1 one in the classes the client @@ -165,7 +174,7 @@ def test_700_table_holds_the_changed_classes_only(): assert set(table_700) - set(table_681) == (set(_VERSIONED_BODIES) - { "Sekai.Mysekai.MysekaiInfoOption1Page", "Sekai.Mysekai.Inventory.CraftPreview", - }) | set(jd.photo_layout.DECODERS) + }) | set(jd.photo_layout.DECODERS) | BULK_CLASSES for name, body in _VERSIONED_BODIES.items(): assert table_700[name].__name__ == body for name in set(table_681) & set(table_700): @@ -184,3 +193,22 @@ def test_scroll_rect_tolerance_reads_after_the_681_chain(): fields, remaining = decode("6.8.1", "Sekai.UI.CustomScrollRect", raw) assert remaining == 4 assert "normalizedPositionTolerance" not in fields + + +def test_bulk_checkbox_and_reward_cell_field_order(): + fields, left = decode("7.0.0", "Sekai.UIPartsCheckBox", + pptr(41) + b4(True) + pptr(42)) + assert left == 0 and fields == {"checkObj": (0, 41), "check": True, "toggle": (0, 42)} + fields, left = decode("7.0.0", "Sekai.UIPartsThumbnailRewardListCell", + f32(24) + f32(30) + i32(2) + pptr(43) + pptr(44) + pptr(45)) + assert left == 0 + assert fields == {"sizeX": 24, "sizeY": 30, + "thumbnailRoots": [(0, 43), (0, 44)], "nameText": (0, 45)} + + +def test_stock_scroll_rect_ends_before_custom_extension(): + raw = scroll_rect(1000) + fields, left = decode("7.0.0", "UnityEngine.UI.ScrollRect", raw) + assert left == 24 + assert fields["m_Content"] == (0, 1000) + assert fields["m_Viewport"] == (0, 1001) From fbe13d3bc8068537ef351d3b22731a70f17e8845 Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:45:45 +0800 Subject: [PATCH 15/15] fix(pack): include bulk gauge resources in published catalogs --- src/pack/groups.py | 2 +- tests/test_asset_architecture_v2.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/pack/groups.py b/src/pack/groups.py index b671393..74b39f3 100644 --- a/src/pack/groups.py +++ b/src/pack/groups.py @@ -61,7 +61,7 @@ "avatar", "avatar-parts", "camera", "cutscene-timeline", "emoticons", "fixture-areas", "fixture-attach", "fixture-interface", "fixture-meshes", "fixture-models", "fixture-gimmick", "fixture-particles-v2", "fixture-talks", "fixture-timeline", "perf-animations", - "phenomena", "site", "ui", "ui-layout-v2", "ui-jp", + "phenomena", "site", "ui", "ui-layout-v2", "ui-jp", "ui-bulkharvest", "field-camera", "house-transition", "live-transitioner", "effect-resources", "graphics-config", "layout-grid", "site-action-effects", "site-common-action-effects", "site-harvest-object-effects", diff --git a/tests/test_asset_architecture_v2.py b/tests/test_asset_architecture_v2.py index b3efd0b..c89950c 100644 --- a/tests/test_asset_architecture_v2.py +++ b/tests/test_asset_architecture_v2.py @@ -710,6 +710,25 @@ def test_actor_export_plans_progress_receipts_and_backups_never_enter_runtime(as assert packages(before) == packages(after) +def test_bulk_gauge_document_and_textures_reach_the_published_catalogue(assets, tmp_path): + required = { + "ui-bulkharvest/gauge/fx_mysekai_bulk_harvest_gauge_charge.json": + b'{"geometry":null,"textures":["textures/pulse.png"]}', + "ui-bulkharvest/gauge/textures/pulse.png": b"synthetic gauge texture", + } + for name, data in required.items(): + path = assets / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + evidence = assets / "ui-bulkharvest/gauge/export-report.json" + evidence.write_bytes(b"private extraction evidence") + assert required.keys() <= runtime_paths(assets).keys() + catalog = build_groups(assets, tmp_path / "store", "7.0.0", region="jp") + published = {path for package in catalog["packages"] for path in package["paths"]} + assert required.keys() <= published + assert evidence.relative_to(assets).as_posix() not in published + + def test_verifier_shares_successful_blob_checks_only_within_one_call(assets, tmp_path, monkeypatch): import pack.verify as verifier out = tmp_path / "store"