diff --git a/README.md b/README.md index 2ea8bb3..ab86cf8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # moly-root +## 可复用的提取任务与文本产品 + +`moly source` 记录输入快照；`moly export talks|text-catalog|official-text|text-runtime` +通过有版本的任务提取对话、维护文本身份、匹配其他区服官方文本并输出按脚本加载的显示分片。 +`moly depot plan|run|verify|layout|gc` 管理任务复用与校验，复用现有 pack 存储。 +旧的平面命令继续可用。使用方式、缓存边界与源目录保留要求见 [提取管线](docs/pipeline.md)。 + ## 玩家布局导入资源 从对应区服的主表和已解密家具包生成导入目录及配色贴图： diff --git a/docs/jp-runtime.md b/docs/jp-runtime.md index fa27331..022e72b 100644 --- a/docs/jp-runtime.md +++ b/docs/jp-runtime.md @@ -1,5 +1,22 @@ # Japanese runtime resources +NPC tutorial poses are exported from the client Resources index with the same +canonical position/quaternion basis as characters and sites: + +```sh +moly export tutorial-placements --apk base.apk --apk asset-pack.apk \ + --client-version 7.0.0 --out-dir runtime/tutorial \ + --processor-input source/tutorial-processor.json +``` + +The optional processor input contains source-qualified introductory poses that +are defined by client code rather than serialized placements. Its identity and +canonical coordinates are checked before writing. Section progress and other +server state remain inputs to the runtime; they are not invented by extraction. +The photo-shot UI export also includes its two dynamically loaded call effects, +each restricted to its own referenced objects so identical internal node names +from neighboring prefabs cannot collide. + The built-in JP commands follow the public asset-discovery protocol documented by `sssekai`. Install the optional protocol dependencies with `pip install -e '.[jp]'`. Discovery and downloads happen only when requested. diff --git a/docs/pipeline.md b/docs/pipeline.md new file mode 100644 index 0000000..9204863 --- /dev/null +++ b/docs/pipeline.md @@ -0,0 +1,178 @@ +# Reproducible extraction tasks + +The extraction pipeline follows the same separation as +[nnnotes](https://github.com/MetaSekaiLab/nnnotes): versioned sources, reusable +tasks, immutable results, and product layouts. The existing Moly pack v2 +contracts and logical runtime paths remain the publication boundary. + +`cli` parses options and calls library code. `source` records and checks input +bytes. `pipeline` composes the existing domain readers, text identity registry, +official-text matcher and display products. `pack` owns the shared blob store +and release retention. Legacy domain exports remain available; they have not +all been converted into cached tasks. + +## Sources and plans + +```sh +moly source record --region jp --version VERSION \ + --input talks=inputs/talk.bundle --input constants=inputs/lib.bundle \ + --out source.json +moly source verify source.json + +moly export talks --master inputs/master --bundle inputs/talk.bundle \ + --lib inputs/lib.bundle --depot asset-store --out extracted \ + --plan --emit-task talks-task.json +moly depot run --depot asset-store --task talks-task.json +``` + +The source receipt and task descriptions contain local input locators. Keep +these in build storage, outside runtime publication. Copying them to another +machine requires replacing their locators with paths to the same bytes; this +does not change the task key. Source receipts identify a caller-supplied region +and version and do not authenticate a publisher or discover new game files. + +`--family fixture` selects the existing furniture dialogue reader. The default +includes solo, birthday and ordinary multi-character conversations. Both use +the existing script and constant readers. The Unity fallback version is an +explicit parameter, not evidence of the actual bundle's engine version. + +`moly export communication-actions --master inputs/master --bundle inputs/communication.bundle +--depot asset-store --out extracted` exports the separate per-character +communication motion loops. It retains participant conditions and random +branches; these scripts contain presentation actions rather than spoken text. +The manifest-driven extractor also routes this family automatically. + +A task key hashes its stage/version, subject, normalized parameters, actual +input hashes and sizes, used implementation/dependency identifiers, and +consumed context. Locators and output directories do not affect it. Source +module hashes distinguish unreleased edits as well as installed versions. +`depot plan --previous old-task.json` explains which key components changed. + +`depot run` rejects incompatible installed stages, changed input bytes, corrupt +cached outputs and forged task keys. It verifies and freezes inputs before +calling readers. `--force` recomputes a result and refuses different output for +the same key. Failures leave no committed result. This first implementation +runs one task at a time under the store's existing publication lock. + +## Text products + +```sh +moly export text-catalog --corpus extracted/talks.json \ + --corpus extracted/fixture-talks/talks.json --region jp \ + --source-revision SOURCE_REVISION --previous prior/text-source.json \ + --depot asset-store --out text-jp +moly export official-text --source text-jp/text-source.json \ + --donor text-cn/text-source.json --locale zh-CN \ + --depot asset-store --out official +moly export text-runtime --source text-jp/text-source.json \ + --overlay official/official-zh-CN.json --depot asset-store --out display +``` + +Omit `--previous` only for the first catalogue. The previous source catalogue +is the durable identity registry used for future updates; preserve it and its +retired history. Text identities and source-change handling are described in +[text localization](text-localization.md). + +`text/index.json` is a `moly-text-runtime/1` root. Hashing a script key as UTF-8 +SHA-256 and taking the first two hex digits selects one of at most 256 routing +buckets. Each `moly-text-routes/1` bucket maps script keys to a display source. +Every reference carries `path`, `sha256` and +`bytes`. Objects live at `text/objects/.json`; discovery is written last. + +Display sources carry only original text, identities, source hashes and current +step bindings. A script's linked tweets are in the same display source. The +complete registry, editor context and retired history are not downloaded for +each playback. Shards have a 2 MiB bound. Unchanged scripts retain identical +bytes when another source script or translation changes. The source root never +contains a translation revision. Publish only `text/` with the resource snapshot. + +Official overlays are a separate tree at `translations///`. +Its `latest.json` (`moly-text-publication/1`) and archived +`revisions/.json` use the same 256-bucket routing. Translation buckets +(`moly-text-translation-routes/1`) map script keys directly to overlay references; +all references are relative to that locale directory. Publish this tree +independently. Editing translations changes neither the source index nor the +resource snapshot. Sessions pin a publication, immutable objects are reusable, +and source ID/hash compatibility is checked per line, not by requiring the +whole source registry revision to match. Retain historical publication roots +and their objects for playback pins and rollback. + +Catalogue browsing uses optional `searchBuckets`: 16 shards selected by the +first hash digit. Source rows are `[id, sourceHash, kind, text]`; translation +rows are `[id, sourceHash, text, source]`, grouped by script. These compact +indexes let hosts translate titles and search all dialogue without fetching +each playback document. Source and translation indexes remain in their +respective publications. A translation edit changes its leaf, route and search +shard; unrelated shards and all resource snapshot bytes remain identical. + +Prepare official text for the existing S3 uploader independently: + +```sh +python tools/deployment/prepare_translations.py \ + --publication display/translations/jp/zh-CN --output translation-staging \ + --object-prefix sekai-extra-assets/ --url-prefix /moly/ +``` + +Use the resulting `upload-manifest.json` with `upload_s3.py` and the existing +destination options. `--scope stable` selects immutable objects and the archived +revision root. After these have been uploaded and verified, `--scope mutable +--mutable-path translations/jp/zh-CN/latest.json --allow-mutable-update` selects +only the current pointer. Replacing an existing pointer also requires +`--expect-previous-sha256` for the exact previous bytes. Dry-run remains the +default; `--execute --exclusive-writer` explicitly enables writes. Retain old +objects and roots for rollback; ordinary resource deployment excludes this tree. +This preparation path handles text-only publications. Fonts use the independent +catalogue described below rather than being copied into translation revisions. + +These exports prepare inputs for the editorial and playback systems. They do +not publish translations or grant approval to drafts. + +## Independent font catalogue + +An operator prepares the approved open font once. Translators continue to enter +text; they do not upload font files. Coverage is read from each supplied font's +character map, and the base font is identified by the exact UI font hash: + +```sh +pip install 'moly-root[fonts]' +moly export text-fonts --base-font UI.subset.ttf --font OPEN-FONT.ttf \ + --license OFL-1.1.txt --out-dir font-publication/fonts +python tools/deployment/prepare_fonts.py --publication font-publication/fonts \ + --output font-staging --object-prefix sekai-extra-assets/ --url-prefix /moly/ +``` + +`fonts/latest.json` and `fonts/revisions/.json` contain the +`moly-text-fonts/1` catalogue, base-font coverage, and immutable font and license +references under `fonts/objects/`. Upload stable objects first, then use the +existing uploader's explicit mutable scope for `fonts/latest.json`, checking +the previous pointer hash when replacing it. No font object changes when a +translator changes text. Retain historical catalogues and referenced objects. +Set NextTrans's `MYSEKAI_FONT_CATALOG_URL` to the public catalogue so its coverage +check and the stage use the same approved fonts. This command only prepares +files and does not deploy or configure the translation service. + +## One store, two kinds of roots + +```text +asset-store/ + blobs/ab/.bin extraction bytes (existing identity codec) + tasks/.json committed moly-result/1, with verified artifacts + packages/ existing release packages + catalogs/ existing retained releases + channels/ existing release selectors +``` + +Task results contain key parts and artifact records, never local input paths +or timings. Both extraction results and retained release catalogs protect +their referenced blobs. `moly depot verify --depot asset-store` verifies all +task results; `python -m pack.verify --out asset-store` verifies release roots +and task roots together when a release exists. `moly depot gc` uses the same +collector as `pack.gc`; it reports only unless `--apply` is passed. It refuses +collection on corrupt roots or an incomplete release layout. Task history is +retained; this interface does not implicitly retire prior source registries. + +`depot layout --key KEY --out DIRECTORY` copies verified artifacts into an +editable output directory, separate from the store. It never hard-links that +directory to shared immutable blobs. Use the existing pack/group publication +workflow for runtime releases. Content/representation identity, JSON number +semantics, existing domain schemas and release channels are unchanged. diff --git a/docs/site-edit-pick.md b/docs/site-edit-pick.md new file mode 100644 index 0000000..f93c8ef --- /dev/null +++ b/docs/site-edit-pick.md @@ -0,0 +1,34 @@ +# Site editor picking surface + +The site editor raycasts the scene colliders tagged `RaycastTargetGroup`. +Export them from the scene bundle and its matching client APK: + +```sh +moly export site-edit-pick \ + --apk client.apk \ + --bundle decrypted/mysekai__site__field__home \ + --out assets/site/edit-pick/home.json +``` + +`--apk` can be repeated for split APKs. The APK must supply +`assets/bin/Data/globalgamemanagers`; its `TagManager.tags` determines the numeric +tag. Use the APK from the same client version as the bundle. The exporter records +the source bundle hash, TagManager file hash and object identifiers; these prove +the bytes supplied but do not infer whether two independently supplied inputs +belong to the same release. + +The version 1 sidecar retains each tagged MeshCollider's enabled, trigger and +convex flags, GameObject layer, root-to-leaf active flags and local TRS, authored +vertex positions and triangle winding. Coordinates stay Unity left-handed, +Y-up; no scene placement, reflection, visual geometry or navigation mesh is +substituted. Triangle indices include each submesh's base vertex. + +Version 1 requires MeshCollider triangle geometry and local pointers in the +collider's serialized file. Missing tags, unresolved pointers, other collider +types, unsupported topology or invalid geometry fail before replacing an +existing output. A disabled collider or inactive ancestor remains represented +with its source flag; the runtime decides whether it can be hit. + +Add the resulting `site/edit-pick/.json` to the resource manifest and use +the existing resource publication workflow. This command only writes the +explicit output file; it does not change a source snapshot or upload resources. diff --git a/docs/text-localization.md b/docs/text-localization.md new file mode 100644 index 0000000..433d1af --- /dev/null +++ b/docs/text-localization.md @@ -0,0 +1,60 @@ +# Display text and ordinary multi-character conversations + +`moly talks` also exports ordinary conversations between two to five characters +under `commonSomeCharacter`. These talks have no fixture anchor. They retain +participant slots, source association rows, conditions, pre-actions, tweets, +script steps and voice cues. The runtime chooses the initiating character; +association metadata does not fix that conversation's leader. + +The audio corpus reader includes this block. Existing solo `units` and +`birthdayParty` blocks keep their own semantics. A consumer must explicitly +support the new family before it can play these conversations. + +## Text catalogues + +Generate a translation source from caller-provided extracted documents: + +```sh +moly text-catalog --corpus extracted/talks.json \ + --corpus extracted/fixture-talks/talks.json \ + --region jp --source-revision SOURCE_REVISION --out text-source.json +``` + +On the next source update pass `--previous text-source-previous.json`. Keep that +previous catalogue in durable versioned storage: it is the registry that +preserves text identities across updates. The output identifies scripts and +display units, with original text, source hashes, speakers, voice context and +bindings to the current script steps. Step numbers are bindings, not identity. + +Unique voice anchors preserve identity when the text changes; unchanged text +can also retain an unambiguous identity. Changes invalidate the old source hash. +Ambiguous repeated lines after a source edit are retired and allocated new IDs +for review. A display-only label does not replace the original control label. +Tweets have their own master identities because multiple talk records can share +a script but reference different tweets. + +The catalogue may contain original game text and derived context. It belongs in +the caller's asset/content storage, not in this source repository. + +## Official translations + +Generate a regional official overlay from two catalogues: + +```sh +moly official-text --source jp-text-source.json --donor cn-text-source.json \ + --locale zh-CN --out official-zh-CN.json +``` + +The matcher requires compatible script identity, participant sets and per-line +speaker/voice context. A same-numbered talk is not enough. Unanchored or +incompatible lines are listed in `issues` for explicit review. A tweet also +needs matching source metadata and script associations. Identical or empty +donor text is not silently labelled a translation. + +The overlay contains plain text, source IDs/hashes and donor provenance. It +never changes executable steps, voice cues, timing or region assets. A text +consumer can prefer matching official text, then a published community +translation, then the original. Source region and display locale are separate. + +These commands prepare source catalogues and official candidates. They do not +publish a service, grant editorial approval, or verify runtime playback. diff --git a/pyproject.toml b/pyproject.toml index 3cee376..5ddc337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ ] [project.optional-dependencies] +fonts = ["fonttools>=4.60,<5"] deployment = [ "brotli>=1.2,<2", "oss2>=2.19,<3", @@ -24,7 +25,7 @@ jp = [ ] [project.scripts] -moly = "core.cli:main" +moly = "cli.main:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/chara/characters.py b/src/chara/characters.py index 95d8fb2..e3175f7 100644 --- a/src/chara/characters.py +++ b/src/chara/characters.py @@ -88,6 +88,69 @@ def one(self, type_name): return objs[0] +def read_animation_controller(assets): + """Exact clip membership of the character's controller-bearing Animator. + + Auxiliary Animators without controllers are present in character packages. + A missing controller stays null; multiple concrete owners are ambiguous. + Clip references retain their serialized order and source identities, so a + consumer can join names and authored timing from the shared motion index + without treating its union of clips as this particular actor's repertoire. + """ + def pointer(owner, value, *, allow_null=False): + if not isinstance(value, dict): + raise ValueError('character animation reference is not a PPtr') + file_id, path_id = value.get('m_FileID'), value.get('m_PathID') + if (type(file_id) is not int or file_id < 0 or type(path_id) is not int + or not -(1 << 63) <= path_id < (1 << 63)): + raise ValueError('character animation reference has invalid identifiers') + if path_id == 0: + if allow_null: + return None + raise ValueError('character controller has a null animation clip') + if file_id == 0: + file = owner.assets_file.name + elif file_id <= len(owner.assets_file.externals): + file = owner.assets_file.externals[file_id - 1].path.replace('\\', '/').rsplit('/', 1)[-1] + else: + raise ValueError('character animation reference has no external file') + if not isinstance(file, str) or not file: + raise ValueError('character animation reference has no source file') + return {'file': file, 'pathId': str(path_id)} + + candidates = [] + for obj in assets.objects: + if obj.type.name != 'Animator': + continue + ref = obj.read_typetree().get('m_Controller') + identity = pointer(obj, ref, allow_null=True) + if identity is not None: + candidates.append((obj, ref, identity)) + if not candidates: + return None + if len(candidates) != 1: + raise ValueError('character needs one controller-bearing Animator') + animator, ref, identity = candidates[0] + controller = assets.deref(animator, ref) + if controller is None or controller.type.name != 'AnimatorController': + raise ValueError('character Animator requires its actual AnimatorController') + source = {'file': controller.assets_file.name, 'pathId': str(controller.path_id)} + if source != identity: + raise ValueError('character Animator controller identity mismatch') + references = controller.read_typetree().get('m_AnimationClips') + if not isinstance(references, list): + raise ValueError('character controller has no animation clip table') + clips = [pointer(controller, value) for value in references] + if len({(value['file'], value['pathId']) for value in clips}) != len(clips): + raise ValueError('character controller duplicates an animation clip') + return { + 'schema': 'moly-character-animation-map/1', + 'animator': {'file': animator.assets_file.name, 'pathId': str(animator.path_id)}, + 'controller': source, + 'clips': clips, + } + + def read_scene(assets): """The instantiable prefab transform tree. @@ -319,8 +382,16 @@ def read_renderers(assets, tr_index): raise KeyError("renderer bone outside the prefab tree") bones.append(bi) go = gos.get(go_pid) + bounds = tt.get('m_AABB') + if bounds is None: + raise ValueError('skinned renderer has no authored bounds') + root_bone = tr_index.get(tt.get('m_RootBone', {}).get('m_PathID', 0)) out.append({"mesh": mesh_obj, "materialNames": mats, "materialObjs": mat_objs, "bones": bones, "node": node, + "bounds": {"node": node, "rootBone": root_bone, + "center": list(unity_to_gltf_pos([bounds['m_Center'][k] for k in ('x', 'y', 'z')])), + "extents": [bounds['m_Extent'][k] for k in ('x', 'y', 'z')], + "updateWhenOffscreen": bool(tt.get('m_UpdateWhenOffscreen', False))}, "rendererName": go["m_Name"] if go else ""}) return out @@ -483,6 +554,29 @@ def chan(node, path, data, dim): return report +def head_ik_axes(avatar, nodes): + """Bind humanoid look-at axes to the emitted skeleton in canonical space.""" + human = avatar['human'] + def bone(slot): + index = human['humanBoneIndex'][slot] + if index < 0: + return None + source = human['nodes'][index] + matches = [i for i, node in enumerate(nodes) if node['name'] == source['name']] + if len(matches) != 1 or source['axesId'] < 0: + raise ValueError(f'look-at bone has no unique skeleton/axes binding: {source["name"]}') + axes = human['axes'][source['axesId']] + def quaternion(value): + # Reflection of X preserves quaternion X/W and negates Y/Z. + return [value[0], -value[1], -value[2], value[3]] + return {'node': matches[0], 'name': source['name'], + 'preQ': quaternion(axes['preQ']), 'postQ': quaternion(axes['postQ'])} + head, neck = bone(11), bone(10) + if head is None: + raise ValueError('humanoid avatar has no head binding') + return {**head, 'neck': neck} + + def extract_character(bundle, out_dir, name, sampled=None, aux=(), unit_id=None, facial=None, atlas_cell=DEFAULT_ATLAS_CELL, with_cloth=True): @@ -491,6 +585,7 @@ def extract_character(bundle, out_dir, name, sampled=None, aux=(), verification numbers (all verifications also raise on hard violations). """ assets = CharacterAssets(bundle, aux) + animation_controller = read_animation_controller(assets) av_tt = assets.one("Avatar").read_typetree() rig = Rig(rig_doc(av_tt)) nodes, tr_index = read_scene(assets) @@ -788,6 +883,9 @@ def wm(i): "mouthAtlas": _atlas_doc(mouth_tex.m_Width, mouth_tex.m_Height, atlas_cell), "cloth": cloth_rig, "anchors": read_anchors(assets, tr_index, nodes), + "headIk": head_ik_axes(rig.doc, nodes), + "animationController": animation_controller, + "rendererBounds": [{**renderer['bounds'], 'name': renderer['rendererName']} for renderer in renderers], "textures": [tex_paths[n] for n in sorted(pngs)], # materials[i].name's contract is "the glTF material's own name", # matching the site domain's convention (there it holds the real diff --git a/src/chara/common_talks.py b/src/chara/common_talks.py new file mode 100644 index 0000000..4cb7a7e --- /dev/null +++ b/src/chara/common_talks.py @@ -0,0 +1,114 @@ +"""Ordinary conversations between multiple characters, without a fixture. + +Membership follows the source's IsCommonSomeCharacterTalk predicate: there +are two to five nonzero group members, no fixture-id condition, and a row in +the some-character association table. Association rows are preserved as +metadata; the runtime's initiating character leads the resulting plan. +Scripts share the ordinary talk parser and voice-bank rule. +""" +from collections import Counter, defaultdict + + +def select_common_talks(master): + groups = {} + for row in master.table("mysekaiGameCharacterUnitGroups"): + key = row["id"] + if key in groups: + raise ValueError(f"duplicate character group {key}") + groups[key] = [row[f"gameCharacterUnitId{i}"] for i in range(1, 6) + if row.get(f"gameCharacterUnitId{i}")] + conditions = master.condition_types() + candidates = [] + for talk in master.table("mysekaiCharacterTalks"): + members = groups.get(talk["mysekaiGameCharacterUnitGroupId"]) + if members is None: + raise ValueError(f"talk {talk['id']}: missing character group") + if not 2 <= len(members) <= 5: + continue + kinds = conditions.get(talk["mysekaiCharacterTalkConditionGroupId"]) + if kinds is None or any(not isinstance(kind, str) for kind in kinds): + raise ValueError(f"talk {talk['id']}: unresolved conditions") + # IsFixtureActionTalk tests this exact condition, not a string prefix + # and not the wider furniture predicate used by the solo selector. + if "mysekai_fixture_id" not in kinds: + candidates.append((talk, members)) + if not candidates: + return [], {"candidates": 0, "selected": 0, "withoutAssociation": []} + associations = defaultdict(list) + for row in master.table("mysekaiCharacterTalkSomeCharacterTalks"): + associations[row["mysekaiCharacterTalkId"]].append(row) + selected, absent = [], [] + for talk, members in candidates: + links = associations.get(talk["id"]) + if not links: + absent.append(talk["id"]) + continue + selected.append({"talk": talk, "unitIds": members, "associations": links}) + return selected, {"candidates": len(candidates), "selected": len(selected), + "withoutAssociation": absent} + + +def common_talks(master, assets, tables, scalars): + """Build the commonSomeCharacter block of the ordinary talk document.""" + from chara.talks import (_asset_named, _tweet, constant_census, parse_script, + TALK_VOICE_BANK) + from perf.fixture_talks import classify_ops + + selected, selection = select_common_talks(master) + rows, scripts, operations = [], [], Counter() + conditions = master.condition_types() + payloads = master.condition_entries() + tweets = master.tweets() + pre_actions = {} + for row in master.table("mysekaiCharacterTalkPreActions"): + pre_actions.setdefault(row["mysekaiCharacterTalkId"], row) + for item in selected: + source = item["talk"] + lua = source["lua"] + name = lua if lua.endswith(".lua") else lua + ".lua" + script = _asset_named(assets, name) + if script is None: + raise LookupError(f"talk {source['id']}: script asset not found: {name}") + _, unknown = classify_ops(script, name) + if unknown: + raise ValueError(f"talk {source['id']}: unknown script operations: {dict(unknown)}") + parsed = parse_script(script, tables, scalars, script_name=name) + pre_action = pre_actions.get(source["id"]) + tweet_id = (pre_action or {}).get("mysekaiCharacterTalkTweetId") + if tweet_id and tweet_id not in tweets: + raise LookupError(f"talk {source['id']}: tweet not found: {tweet_id}") + group = source["mysekaiCharacterTalkConditionGroupId"] + rows.append({ + "talkId": source["id"], "lua": lua, + "unitGroupId": source["mysekaiGameCharacterUnitGroupId"], + "unitIds": item["unitIds"], + "someCharacterTalks": item["associations"], + "siteGroupId": source["mysekaiSiteGroupId"], + "termId": source["mysekaiCharacterTalkTermId"], + "conditionGroupId": group, "conditions": conditions[group], + "conditionValues": payloads[group], + "preAction": pre_action, "tweet": _tweet(tweets.get(tweet_id)), + "voiceBank": TALK_VOICE_BANK.format(lua.removesuffix(".lua")), + "voices": parsed["voices"], "steps": parsed["steps"], + }) + scripts.append(script) + operations.update(parsed["operations"]) + constants = constant_census(scripts, tables) + summary = { + "talks": len(rows), "steps": sum(len(row["steps"]) for row in rows), + "voiceCues": sum(len(row["voices"]) for row in rows), + "uniqueVoiceCues": len({cue for row in rows for cue in row["voices"]}), + "operations": dict(operations), "selection": selection, + "constants": constants, + } + return { + "version": 1, + "semantics": { + "membership": "Non-fixture talks with two to five group members and a some-character association.", + "mainCharacter": "Chosen by the runtime: the initiating character leads the plan; association metadata does not fix its leader.", + "unitIds": "Nonzero character group slots, in source slot order.", + "preAction": "The source pre-action row, or null; its tweet is resolved separately.", + "fixture": "This family has no fixture anchor; it requires common multi-character runtime support.", + }, + "talks": rows, "summary": summary, + }, summary diff --git a/src/chara/communication_actions.py b/src/chara/communication_actions.py new file mode 100644 index 0000000..719ccda --- /dev/null +++ b/src/chara/communication_actions.py @@ -0,0 +1,107 @@ +"""Per-character communication performances, distinct from spoken dialogue. + +The script's lifetime loop and participant-gated random prefix are retained. +Channel steps use the existing performance reader and its source library's +argument order. No dialogue, voice or translated text is invented here. +""" +from collections import Counter +import re + +from core.jsonio import write_json +from core.master import Master +from chara import alone_actions +from chara.talks import _asset_named, _calls, _resolve, _strip_comments, _text_assets + +PACKAGE = "mysekai__some_character_communication" +TABLE = "mysekaiCharacterTalkTogetherCommunications" + + +def parse_program(script, tables, scalars, defaults): + text = _strip_comments(script).strip() + loop = re.fullmatch(r"while\s+enable_communication\(\)\s+do\s*(.*?)\s*end", text, re.S) + if not loop: + raise ValueError("unsupported communication lifetime loop") + body = loop[1] + branches, random = [], None + prefix = re.match(r"\s*rand\s*=\s*math\.random\(0,\s*99\)\s*", body) + if prefix: + random = {"min": 0, "max": 99} + body = body[prefix.end():] + branch = re.match(r"if\s+is_communicating\((\w+\.\w+)\)\s+and\s+rand\s*<\s*(\d+)\s+then(.*?)" + r"elseif\s*\(\s*rand\s*<\s*(\d+)\s*\)\s*then(.*?)\bend\b", body, re.S) + if not branch: + raise ValueError("unsupported communication random branch") + participant, _ = _resolve(branch[1], tables, scalars) + if not isinstance(participant, int): + raise ValueError("unresolved communication participant") + branches = [{"condition": {"participant": participant, "randomBelow": int(branch[2])}, "body": branch[3]}, + {"condition": {"participant": None, "randomBelow": int(branch[4])}, "body": branch[5]}] + body = body[branch.end():] + + def steps(fragment): + if re.search(r"\b(?:if|elseif|else|end|while|for|repeat|function|return)\b", fragment): + raise ValueError("unhandled communication control flow") + allowed = set(alone_actions.CALL_OPS) | {"wait_time"} + calls = list(_calls(fragment)) + # _calls deliberately scans the shared talk vocabulary. Also account + # for every lexical function name so a future command cannot vanish. + names = re.findall(r"\b([A-Za-z_]\w*)\s*\(", fragment) + if any(name not in allowed for name in names) or len(names) != len(calls): + raise ValueError("unsupported communication operation") + result, elapsed = [], 0.0 + for op, args in calls: + call = ("alone_wait_time(nil, " + ",".join(args) + ")" if op == "wait_time" + else op + "(" + ",".join(args) + ")") + emitted, count = alone_actions._steps(call, tables, scalars, defaults) + if count != 1: + raise ValueError("communication call accounting mismatch") + if op != "wait_time": + who, _ = _resolve(args[0], tables, scalars) + if not isinstance(who, int): + raise ValueError("unresolved communication actor") + for step in emitted: + step["who"] = who + for step in emitted: + step["t"] = elapsed + if step["op"] == "animation" and not isinstance(step["motion"], str): + raise ValueError("unresolved communication motion") + if step["op"] == "wait": + if not isinstance(step["seconds"], (int, float)) or step["seconds"] < 0: + raise ValueError("invalid communication wait duration") + elapsed += step["seconds"] + result.append(step) + return result + + for branch in branches: + branch["steps"] = steps(branch.pop("body")) + tail = steps(body) + if not any(step["op"] == "wait" and isinstance(step["seconds"], (int, float)) and step["seconds"] > 0 + for step in tail): + raise ValueError("communication lifetime loop has no positive wait") + return {"kind": "whileCommunication", "random": random, "branches": branches, "steps": tail} + + +def extract_communication_actions(master_source, bundle, out_path, master_cache=None): + assets = _text_assets(bundle) + defines = _asset_named(assets, "defines.lua") + library = _asset_named(assets, "lib.lua") + if defines is None or library is None: + raise ValueError("communication package is missing its libraries") + tables = alone_actions.parse_constant_tables(defines) + scalars = alone_actions.parse_constant_scalars(defines, tables) + defaults = alone_actions.parse_library_defaults(library) + units, operations = {}, Counter() + for row in Master(master_source, cache_dir=master_cache).table(TABLE): + unit = str(row["gameCharacterUnitId"]) + if unit in units: + raise ValueError("duplicate communication character") + script = _asset_named(assets, row["lua"] + ".lua") + if script is None: + raise ValueError("missing communication script: " + row["lua"]) + program = parse_program(script, tables, scalars, defaults) + units[unit] = {"id": row["id"], "lua": row["lua"], "program": program} + for steps in [program["steps"], *(b["steps"] for b in program["branches"])]: + operations.update(step["op"] for step in steps) + summary = {"units": len(units), "steps": sum(operations.values()), "operations": dict(operations)} + write_json(out_path, {"version": 1, "units": units, "summary": summary}) + return {**summary, "path": str(out_path)} diff --git a/src/chara/registry.py b/src/chara/registry.py index d8a6ed5..33b9a01 100644 --- a/src/chara/registry.py +++ b/src/chara/registry.py @@ -40,6 +40,10 @@ "gameCharacterId: firstName + givenName, skipping absent parts; " "null when the character row is absent — the caller decides how to " "label a nameless unit, this module never invents one"), + "gameCharacter": ("identity.gameCharacterId joined to the gameCharacters row; " + "id and verbatim gender are kept together. Null means the " + "source row is absent; gender is not inferred from unit, " + "motion family, model name, or available animation files"), "storedVsRuntime": ("the runtime divides pauseMilliSeconds, " "changeMotionMilliSeconds, walkSpeed and runSpeed by 1000; " "both forms are given and the runtime form is the one to use"), @@ -100,11 +104,14 @@ def build_registry(master, unit_ids, motion_library_index=None, client_configs=_ ("soloAction", solo.get(unit))) if row is None] if gaps: missing[str(unit)] = gaps - name = _display_name(game_characters.get(ident_row["gameCharacterId"]) - if (game_characters and ident_row) else None) + character_id = ident_row.get("gameCharacterId") if ident_row else None + character_row = game_characters.get(character_id) if game_characters is not None else None + name = _display_name(character_row) entry = { "unitId": unit, "name": name, + "gameCharacter": ({"id": character_id, "gender": character_row.get("gender")} + if character_row is not None else None), "identity": ({f: ident_row.get(f) for f in IDENTITY_FIELDS} if ident_row else None), "locomotion": None, @@ -160,6 +167,7 @@ def build_registry(master, unit_ids, motion_library_index=None, client_configs=_ "requested": len(set(unit_ids)), "withIdentity": sum(1 for c in characters.values() if c["identity"]), "withName": sum(1 for c in characters.values() if c["name"]), + "withGameCharacter": sum(1 for c in characters.values() if c["gameCharacter"] is not None), "withLocomotion": sum(1 for c in characters.values() if c["locomotion"]), "withSoloAction": sum(1 for c in characters.values() if c["soloAction"]), "missing": missing, diff --git a/src/chara/talks.py b/src/chara/talks.py index eb120fc..e835230 100644 --- a/src/chara/talks.py +++ b/src/chara/talks.py @@ -610,7 +610,9 @@ def parse_script(text, tables, scalars=None, script_name=None, defaults=None, def _text_assets(bundle): """Return ``{TextAsset name: decoded script text}`` from a Unity bundle.""" out = {} - for obj in UnityPy.load(bundle).objects: + # These small script bundles are consumed in one pass. Loading their + # bytes avoids retaining an open source file through Unity object cycles. + for obj in UnityPy.load(Path(bundle).read_bytes()).objects: if obj.type.name != "TextAsset": continue tree = obj.read_typetree() @@ -771,6 +773,29 @@ def birthday_talks(master, assets, tables, scalars): return block, dict(report, status="present") +def tutorial_talks(master, assets, tables, scalars): + """Tutorial IDs select authored scripts; they do not enter the talk lottery.""" + try: + rows = master.table('mysekaiTutorialTalks') + except MissingTable: + return {'status': 'absent', 'rows': [], 'talks': {}}, {'status': 'absent', 'scripts': 0} + scripts = {} + for row in rows: + lua = row['lua'] + if lua in scripts: + continue + name = lua if lua.endswith('.lua') else lua + '.lua' + script = _asset_named(assets, name) + if script is None: + raise LookupError(f'tutorial script asset not found: {name}') + parsed = parse_script(script, tables, scalars, script_name=name) + scripts[lua] = {'voices': parsed['voices'], 'steps': parsed['steps'], + 'voiceBank': 'mysekai/talk/tutorial/voice/' + lua} + return {'status': 'complete', 'rows': rows, 'talks': scripts}, { + 'status': 'complete', 'scripts': len(scripts), 'rows': len(rows), + 'steps': sum(len(script['steps']) for script in scripts.values())} + + def _semantics(source): return { "selection": ( @@ -911,6 +936,14 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, # its exact serialized form. birthday, birthday_report = birthday_talks(master, assets, tables, scalars) doc["birthdayParty"] = birthday + # Multi-character ordinary talks share the source package and parser, + # but have their own membership and runtime plan. Do not put them into + # any participant's solo lottery or invent a fixture for them. + from chara.common_talks import common_talks + common, common_report = common_talks(master, assets, tables, scalars) + doc["commonSomeCharacter"] = common + tutorial, tutorial_report = tutorial_talks(master, assets, tables, scalars) + doc['tutorial'] = tutorial os.makedirs(os.path.dirname(os.path.abspath(out_path)) or ".", exist_ok=True) with open(out_path, "w", encoding="utf-8", newline="\n") as handle: json.dump(doc, handle, ensure_ascii=False, indent=1, allow_nan=False) @@ -921,6 +954,8 @@ def extract_talks(master_source, talk_bundle, out_path, master_cache=None, "constantSource": constant_source, "constantTables": {key: len(value) for key, value in tables.items()}, "birthdayParty": birthday_report, + "commonSomeCharacter": common_report, + "tutorial": tutorial_report, } diff --git a/src/chara/text_catalog.py b/src/chara/text_catalog.py new file mode 100644 index 0000000..a4afe35 --- /dev/null +++ b/src/chara/text_catalog.py @@ -0,0 +1,262 @@ +"""Versioned display-text catalogues, independent of executable scripts. + +Text IDs survive source revisions through unique voice/context anchors or +unchanged text. Ambiguous repetitions are never silently paired after an +edit. Bindings locate a display operation in a specific script revision; +they are not the translation identity. +""" +from collections import defaultdict +import hashlib +import json + +from core.jsonio import write_json + +SCHEMA = "moly-text-source/1" +REGIONS = {"jp", "cn", "tw", "en", "kr"} + + +def digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest() + + +def corpus_rows(document): + for unit, group in (document.get("units") or {}).items(): + for row in group.get("talks", []): + yield "general", row, [int(unit)] + for row in document.get("talks", []): + yield "fixture", row, row["unitIds"] + for row in (document.get("commonSomeCharacter") or {}).get("talks", []): + yield "common", row, row["unitIds"] + for lua, row in (document.get("birthdayParty") or {}).get("talks", {}).items(): + yield "birthday", {**row, "lua": lua, "talkId": lua}, [] + for lua, row in (document.get("tutorial") or {}).get("talks", {}).items(): + yield "tutorial", {**row, "lua": lua, "talkId": lua}, [] + + +def _display_rows(row): + """Original label tokens remain untouched; their displays get bindings.""" + rows = [] + speaker, unit, pending = "", None, [] + for index, step in enumerate(row["steps"]): + op = step["op"] + if op == "label": + speaker = step["name"] + unit, pending = None, [] + rows.append({"kind": "speaker", "step": index, "text": speaker, + "speaker": speaker, "speakerUnit": None, "cues": []}) + elif op in ("voice", "fixture_voice"): + unit = step.get("who", step.get("fixture")) + pending.append(step["cue"]) + # Labels conventionally precede voice; attach its identity to + # that label's display without changing the label/control token. + for display in reversed(rows): + if display["kind"] == "speaker": + display["speakerUnit"] = unit + display["cues"] = list(pending) + break + if display["kind"] == "text": + break + elif op == "text": + rows.append({"kind": "text", "step": index, "text": step["text"], + "speaker": speaker, "speakerUnit": unit, "cues": list(pending)}) + pending = [] + for i, display in enumerate(rows): + if not isinstance(display["text"], str): + raise ValueError(f"{row['lua']}: display text is not a string") + display["position"] = i + display["sourceHash"] = digest([display["kind"], display["text"], display["speaker"], + display["speakerUnit"], display["cues"]]) + display["anchor"] = digest([display["kind"], display["speakerUnit"], display["cues"], + display.get("tweetId")]) if display["cues"] or display["kind"] == "tweet" else None + return rows + + +def _assign_ids(region, script, current, previous): + old = previous.get("lines", []) if previous else [] + retired_before = previous.get("retiredLines", []) if previous else [] + used, matched = set(), {} + # Unchanged complete display sequence permits exact occurrence mapping. + # A motion insertion changes bindings but not this sequence. + if [r["sourceHash"] for r in old] == [r["sourceHash"] for r in current]: + matched = {i: r for i, r in enumerate(old)} + used = {r["id"] for r in old} + else: + for field in ("anchor", "sourceHash"): + old_by, new_by = defaultdict(list), defaultdict(list) + for r in old: + if r["id"] not in used and r.get(field): + old_by[r[field]].append(r) + for i, r in enumerate(current): + if i not in matched and r.get(field): + new_by[r[field]].append(i) + for key, indices in new_by.items(): + if len(indices) == 1 and len(old_by.get(key, [])) == 1: + match = old_by[key][0] + matched[indices[0]] = match + used.add(match["id"]) + old_ids = {r["id"] for r in old + retired_before} + allocated = set(old_ids) + for i, row in enumerate(current): + if i in matched: + prior = matched[i] + row["id"] = prior["id"] + row["change"] = "unchanged" if row["sourceHash"] == prior["sourceHash"] else "changed" + else: + salt = 0 + while True: + identity = "t_" + digest([region, script, row["kind"], row["anchor"], + row["sourceHash"], i, salt])[:32] + if identity not in allocated: + break + salt += 1 + row["id"] = identity + row["change"] = "new" + allocated.add(identity) + return retired_before + [r for r in old if r["id"] not in used] + + +def build_catalog(documents, *, region, source_revision, previous=None): + if region not in REGIONS or not source_revision: + raise ValueError("a supported source region and explicit source revision are required") + if previous and (previous.get("schema") != SCHEMA or previous.get("region") != region): + raise ValueError("previous text catalogue belongs to another schema or region") + old = {r["key"]: r for r in (previous or {}).get("retiredScripts", [])} + old.update({r["key"]: r for r in (previous or {}).get("scripts", [])}) + scripts, tweet_refs = {}, {} + for document in documents: + for kind, row, members in corpus_rows(document): + key = row["lua"].removesuffix(".lua") + lines = _display_rows(row) + signature = digest(row["steps"]) + content = {"kind": kind, "id": row["talkId"], "unitIds": members} + tweet = row.get("tweet") + if tweet and isinstance(tweet.get("text"), str): + tweet_key = "@tweet/" + str(tweet["id"]) + if tweet_key in tweet_refs and tweet_refs[tweet_key]["tweet"] != tweet: + raise ValueError(f"conflicting source bodies for tweet {tweet['id']}") + ref = tweet_refs.setdefault(tweet_key, {"tweet": tweet, "contents": []}) + tweet_content = {**content, "scriptKey": key} + if tweet_content not in ref["contents"]: + ref["contents"].append(tweet_content) + content = {**content, "tweetKey": tweet_key} + if key in scripts: + existing = scripts[key] + if existing["scriptHash"] != signature: + raise ValueError(f"conflicting source bodies for script {key}") + if content not in existing["contents"]: + existing["contents"].append(content) + continue + retired = _assign_ids(region, key, lines, old.get(key)) + scripts[key] = {"key": key, "scriptHash": signature, "contents": [content], + "lines": lines, "retiredLines": retired, + "displaySequence": [s["op"] for s in row["steps"] if s["op"] in + ("label", "text", "voice", "fixture_voice", "wait_click")], + "speakerIdentity": sorted({str(s.get("who", s.get("fixture"))) + for s in row["steps"] + if s["op"] in ("voice", "fixture_voice")})} + # A script may be referenced by several talk rows with different tweets. + # Tweets have their own master identities, not a script-body identity. + for key, ref in tweet_refs.items(): + tweet = ref["tweet"] + lines = [{"kind": "tweet", "step": None, "tweetId": tweet["id"], + "text": tweet["text"], "speaker": "", "speakerUnit": None, + "cues": [], "position": 0, + "sourceHash": digest(["tweet", tweet]), + "anchor": digest(["tweet", tweet["id"]])}] + retired = _assign_ids(region, key, lines, old.get(key)) + scripts[key] = {"key": key, "scriptHash": digest(tweet), + "contents": ref["contents"], "lines": lines, "retiredLines": retired, + "tweetContext": {k: v for k, v in tweet.items() if k != "text"}} + result = {"schema": SCHEMA, "region": region, "sourceRevision": source_revision, + "scripts": list(scripts.values()), + "retiredScripts": [r for key, r in old.items() if key not in scripts]} + # Import diagnostics (new/changed/unchanged and retired history) are not + # published source content. Reimporting identical bytes keeps a revision. + result["revision"] = digest({ + "schema": SCHEMA, "region": region, "sourceRevision": source_revision, + "scripts": [{k: ([{lk: lv for lk, lv in line.items() if lk != "change"} + for line in v] if k == "lines" else v) + for k, v in script.items() if k != "retiredLines"} + for script in result["scripts"]], + }) + return result + + +def match_official(source, donor, *, locale): + """Pair complete display sequences only when spoken context agrees. + + Same script name is a candidate, never sufficient proof. Require equal + participating units, line kinds and per-line voice/speaker identity. + Lines without a voice anchor require an explicit reviewed mapping. + """ + expected = {"zh-CN": "cn", "zh-TW": "tw", "en-US": "en", "ko-KR": "kr"} + if source.get("schema") != SCHEMA or donor.get("schema") != SCHEMA: + raise ValueError("unsupported source catalogue schema") + if expected.get(locale) != donor["region"]: + raise ValueError("official text locale does not match donor region") + donors = {r["key"]: r for r in donor["scripts"]} + entries, issues = [], [] + for script in source["scripts"]: + other = donors.get(script["key"]) + reason = None + if other is None: + reason = "missing_script" + else: + def members(value): + return sorted({tuple(c["unitIds"]) for c in value["contents"]}) + def spoken(value): + return [(r["kind"], r["speakerUnit"], r["cues"]) for r in value["lines"] if r["kind"] != "tweet"] + if members(script) != members(other): + reason = "participants_differ" + elif script.get("displaySequence") != other.get("displaySequence"): + reason = "display_control_sequence_differs" + elif script.get("speakerIdentity") != other.get("speakerIdentity"): + reason = "speaker_identities_differ" + elif script.get("tweetContext") != other.get("tweetContext"): + reason = "tweet_context_differs" + elif script.get("tweetContext") is not None and sorted( + {c["scriptKey"] for c in script["contents"]} + ) != sorted({c["scriptKey"] for c in other["contents"]}): + reason = "tweet_script_links_differ" + elif spoken(script) != spoken(other): + reason = "display_sequence_differs" + if reason: + issues.append({"script": script["key"], "reason": reason}) + continue + source_lines = script["lines"] + donor_lines = other["lines"] + anchors = defaultdict(int) + for line in source_lines: + anchors[line.get("anchor")] += 1 + for line, official in zip(source_lines, donor_lines): + if line.get("anchor") and anchors[line["anchor"]] > 1: + issues.append({"script": script["key"], "id": line["id"], "reason": "ambiguous_voice_anchor"}) + continue + if line["kind"] != "tweet" and (not line["cues"] or line["speakerUnit"] is None): + issues.append({"script": script["key"], "id": line["id"], "reason": "unanchored_line"}) + continue + if not official["text"].strip() or official["text"] == line["text"]: + issues.append({"script": script["key"], "id": line["id"], "reason": "empty_or_unchanged_text"}) + continue + entries.append({"id": line["id"], "sourceHash": line["sourceHash"], + "text": official["text"], "source": "official_" + donor["region"], + "donor": {"region": donor["region"], "revision": donor["revision"], + "script": other["key"], "id": official["id"], + "sourceHash": official["sourceHash"]}}) + result = {"schema": "moly-text-overlay/1", "sourceRegion": source["region"], + "sourceRevision": source["revision"], "locale": locale, + "entries": entries, "issues": issues} + result["revision"] = digest(result) + return result + + +def export_catalog(paths, out, *, region, source_revision, previous=None): + def read(path): + with open(path, encoding="utf-8") as handle: + return json.load(handle) + result = build_catalog([read(p) for p in paths], region=region, + source_revision=source_revision, previous=read(previous) if previous else None) + write_json(out, result) + return {"scripts": len(result["scripts"]), + "lines": sum(len(r["lines"]) for r in result["scripts"]), "revision": result["revision"]} diff --git a/src/chara/tutorial.py b/src/chara/tutorial.py new file mode 100644 index 0000000..fee56c5 --- /dev/null +++ b/src/chara/tutorial.py @@ -0,0 +1,109 @@ +"""NPC tutorial placements from the client Resources index.""" +import hashlib +import json +import math +from pathlib import Path + +from core.gltf import coordinate_contract, reflect_position, reflect_quaternion +from core.jsonio import write_json +from ui import talk +from ui.player_source import ApkPlayerData, container_target, resource_container, unity_version + +PLACEMENT_CLASS = 'Sekai.Mysekai.MysekaiTutorialCharacterPlacementData' +PLACEMENT_TYPE = '3c921a0c7b9d90b022e9c5428407c181' +PLACEMENT_PREFIX = 'mysekai/tutorial/placement/' + + +def decode_placements(reader): + count = reader.i32() + if count < 0 or count > reader.remaining() // 32: + raise ValueError('tutorial placement count exceeds its serialized object') + return {'datas': [{'unitCharacterId': reader.i32(), 'position': reader.vec3(), + 'rotation': reader.vec4()} for _ in range(count)]} + + +def processor_configuration(path, *, client_version): + """Validate separately supplied client constants; never invent server state.""" + raw = Path(path).read_bytes() + if len(raw) > 65536: + raise ValueError('tutorial processor configuration exceeds its size limit') + value = json.loads(raw) + if not isinstance(value, dict) or set(value) != { + 'schema', 'region', 'clientVersion', 'coordinateContract', 'initialOffstage', + 'initialPlayer', 'afterIntroNpc', 'afterTalkPlayerRotation', + } or value['schema'] != 'moly-tutorial-processor/1' or value['region'] != 'jp' or value['clientVersion'] != client_version: + raise ValueError('tutorial processor configuration has a different source identity') + if value['coordinateContract'] != coordinate_contract()['space']: + raise ValueError('tutorial processor coordinates must already be canonical') + + def vector(data, width): + if not isinstance(data, list) or len(data) != width or any( + type(v) not in {int, float} or not math.isfinite(v) for v in data + ): + raise ValueError('invalid tutorial processor pose') + if width == 4 and not math.isclose(sum(v * v for v in data), 1, abs_tol=1e-4): + raise ValueError('invalid tutorial processor rotation') + + vector(value['initialOffstage'], 3) + vector(value['afterTalkPlayerRotation'], 4) + for name in ['initialPlayer', 'afterIntroNpc']: + pose = value[name] + if not isinstance(pose, dict) or set(pose) != {'position', 'rotation'}: + raise ValueError('invalid tutorial processor transform') + vector(pose['position'], 3) + vector(pose['rotation'], 4) + return value + + +def export_placements(apks, out_dir, *, client_version, processor_input=None): + if client_version != '7.0.0': + raise ValueError('tutorial placement extraction requires a supported source version') + processor = None if processor_input is None else processor_configuration(processor_input, client_version=client_version) + data = ApkPlayerData(apks) + player = data.environment() + player.add(['globalgamemanagers'], follow=False) + manager, container = resource_container(player.env) + selected = [] + for key, pointers in sorted(container.items()): + if not key.startswith(PLACEMENT_PREFIX): + continue + if len(pointers) != 1: + raise ValueError(f'ambiguous tutorial placement resource {key}') + selected.append((key, container_target(player.env, manager, pointers[0]))) + if not selected: + raise ValueError('source has no tutorial placement resources') + player.add([target[0] for _, target in selected]) + if player.missing: + raise ValueError(f'tutorial placements have missing source files: {player.missing}') + scripts = talk.build_monoscript_index(player.env) + placements, sources = {}, [] + for key, (file_name, path_id) in selected: + obj = player.env.files[file_name].objects[path_id] + if talk.resolve_script_class(obj, scripts) != PLACEMENT_CLASS or talk.serialized_type_hash(obj) != PLACEMENT_TYPE: + raise ValueError(f'unrecognized tutorial placement layout: {key}') + decoded = talk.decode_object(player.env, obj, scripts, decoders={PLACEMENT_CLASS: decode_placements}) + if decoded['residual']: + raise ValueError(f'tutorial placement has unread serialized bytes: {key}') + rows = decoded['fields']['datas'] + ids = [row['unitCharacterId'] for row in rows] + if any(value <= 0 for value in ids) or len(set(ids)) != len(ids): + raise ValueError(f'invalid tutorial participant identities: {key}') + for row in rows: + if not all(math.isfinite(value) for value in row['position'] + row['rotation']): + raise ValueError(f'non-finite tutorial pose: {key}') + if not math.isclose(sum(v * v for v in row['rotation']), 1, abs_tol=1e-4): + raise ValueError(f'invalid tutorial rotation: {key}') + placements[key.removeprefix(PLACEMENT_PREFIX)] = [ + {'unitCharacterId': row['unitCharacterId'], + 'position': reflect_position(row['position']), + 'rotation': reflect_quaternion(row['rotation'])} for row in rows] + sources.append({'resource': key, 'serializedFile': file_name, 'pathId': path_id, + 'sourceSha256': hashlib.sha256(obj.get_raw_data()).hexdigest(), + 'typeHash': PLACEMENT_TYPE, 'rows': len(rows), 'residual': 0}) + result = {'schema': 'moly-tutorial-placements/1', 'region': 'jp', 'clientVersion': client_version, + 'unityVersion': unity_version(player.env), 'coordinates': coordinate_contract(), + 'placements': placements, 'sources': sources} + write_json(Path(out_dir) / 'placements.json', result) + if processor is not None: + write_json(Path(out_dir) / 'processor.json', processor) + return result diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..f84057b --- /dev/null +++ b/src/cli/__init__.py @@ -0,0 +1 @@ +"""Command groups; library layers do not import the command line.""" diff --git a/src/cli/__main__.py b/src/cli/__main__.py new file mode 100644 index 0000000..774cc25 --- /dev/null +++ b/src/cli/__main__.py @@ -0,0 +1,3 @@ +from .main import main + +raise SystemExit(main()) diff --git a/src/cli/legacy.py b/src/cli/legacy.py new file mode 100644 index 0000000..8634066 --- /dev/null +++ b/src/cli/legacy.py @@ -0,0 +1,1758 @@ +"""Single command-line entry point for bundle extraction.""" +import argparse +import os +import warnings +import json +import sys +from pathlib import Path + +# Extraction output root; MOLY_EXTRACT_OUT lets a workspace keep it outside the checkout. +DEFAULT_DATA_ROOT = os.environ.get("MOLY_EXTRACT_OUT") or "local-data" + + +PLAYER_DATA_HELP = ("the player data: the container file (assets/bin/Data/data.unity3d), " + "or, for a build without one, each APK whose assets/bin/Data members " + "are its loose player files; repeat for the base APK and the asset pack") + + +def _configure_unity(fallback_version): + import UnityPy.config + if fallback_version: + UnityPy.config.FALLBACK_UNITY_VERSION = fallback_version + + +def _master_source(args): + """Directory or base URL for the caller's master tables, plus a cache dir. + + ``--master`` names a local directory; ``--master-url`` fetches each table by + appending its name to a base URL (with no value, the public default base). + Neither is implied: without one of them nothing is read and nothing is + fetched. + """ + from core.master import DEFAULT_MASTER_URL + url = getattr(args, "master_url", None) + if url is not None: + return url or DEFAULT_MASTER_URL, getattr(args, "master_cache", None) + return getattr(args, "master", None), None + + +def _write_programs(directory, entries, platform=None): + """Write each program body to its own file; return how many were written. + + A body is named by shader, platform and record index rather than by content + hash, so a caller reading a file knows which variant produced it; identical + variants therefore appear more than once, which is the honest shape -- the + census reports the distinct count separately. + """ + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + written = 0 + for entry in entries: + stem = "".join(c if c.isalnum() or c in "-_" else "_" for c in entry["shader"]) + for block in entry["platforms"]: + if platform is not None and block["platform"] != platform: + continue + for row in block["programs"]: + suffix = "glsl" if row["isText"] else "bin" + name = f"{stem}__platform{block['platform']}__record{row['record']}.{suffix}" + (directory / name).write_bytes(row.get("code") or b"") + written += 1 + return written + + +def _extract_failed(report): + """A failed derived artifact is a failed extraction, not a skipped option. + + So is a failed player-data artifact: its input was supplied and it could + not be read, which is a failure of the run, while a player-data artifact + whose input nobody supplied stays ``skipped`` and is named as an input gap. + """ + return bool(report["summary"]["failed"]) or any( + entry.get("status") == "failed" + for entry in report.get("derived", []) + report.get("playerData", [])) + + +def _print_extract_summary(report, out_dir): + s = report["summary"] + print(f"extracted {s['succeeded']}/{s['requested']} bundles -> {out_dir} " + f"(failed {s['failed']}, unsupported {s['unsupported']})") + for entry in report["bundles"]: + if entry["status"] == "failed": + print(f" failed: {entry['bundle']}: {entry['error']}") + for entry in report.get("derived", []) + report.get("playerData", []): + if entry.get("status") == "failed": + print(f" failed artifact: {entry['artifact']}: {entry.get('error', '')}") + programs = entry.get("sourcePrograms") + if programs: + print(f" particle source programs: {programs['status']} " + f"({programs['converted']}/{programs['gles3Variants']} GLES3 variants converted, " + f"unavailable {programs['unavailable']})") + print(f" {programs['note']}") + # An artifact whose input nobody supplied is not a failure of the run, but + # it is not silent either: each one is named with its reason (a failed one + # is already named above). + for gap in (report.get("inputGaps") or {}).get("unaccepted", []): + if gap.get("why") != "failed": + print(f" input gap: {gap['artifact']}: {gap.get('detail', '')}") + print(f"report: {report['report']}") + if str(out_dir) == "local-data": + print("view: python -m http.server 8000 (from the repository root), then open\n" + " http://localhost:8000/examples/viewer/index.html?base=../../local-data") + else: + print("view: serve the repository root over HTTP and open " + "examples/viewer/index.html?base=") + + +def main(argv=None): + from chara.rotation_encoding import DEFAULT_ROTATION_ENCODING, ROTATION_ENCODINGS + # UnityPy warns about asset shapes it does not model. Silencing that is + # this command's choice to make, so it is made here, once, for this + # process. It used to sit at module scope in four extractors, where + # importing one rewrote the warning filters of any program that used this + # package as a library -- including filters that program had set itself. + warnings.filterwarnings("ignore") + ap = argparse.ArgumentParser(prog="moly", description="Extract Unity humanoid assets", + epilog="Grouped workflows: moly source --help; moly export talks --help; moly export text-runtime --help; moly depot --help. Flat exports remain compatible.") + ap.add_argument("--unity-version", default=None) + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("pull", help="download, decrypt, and extract character asset packs") + p.add_argument("--manifest", required=True) + p.add_argument("--out", help="download/decrypt workspace (default: moly-pull-output)") + p.add_argument("--extract-out", default=None, + help="where extraction artifacts go (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") + p.add_argument("--json", action="store_true", + help="print the full JSON report to stdout instead of a summary") + p.add_argument("--asset-base-url", required=True) + p.add_argument("--workers", type=int, default=8) + p.add_argument("--retries", type=int, default=4) + p.add_argument("--master", help="directory of caller-supplied master tables") + p.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append .json to; no value uses the public default base") + p.add_argument("--master-cache", help="where fetched tables are cached") + p.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + p.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") + x = sub.add_parser("extract", help="extract bundles listed in a manifest") + x.add_argument("--manifest", + help="optional bundle list; omitted = select every supported " + "asset-pack bundle found under --bundles") + x.add_argument("--bundles", required=True) + x.add_argument("--out", default=DEFAULT_DATA_ROOT, + help="output directory (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") + x.add_argument("--json", action="store_true", + help="print the full JSON report to stdout instead of a summary") + x.add_argument("--master", help="directory of caller-supplied master tables") + x.add_argument("--player-data", action="append", + help="the APK player data: the container file " + "(assets/bin/Data/data.unity3d), or, for a build " + "without one, each APK whose assets/bin/Data members " + "are its loose player files (repeat: the base APK and " + "the asset pack). It supplies the built-in UI and " + "screen-layer assets that the download path does not " + "carry (container builds only) and the Resources " + "objects: the live transitioner, the effect type " + "table, the graphics configuration, the layout grid") + x.add_argument("--player-types", metavar="DIR", + help="managed assembly stubs of the same player build (a DummyDll " + "directory); the Resources objects read from --player-data " + "(the live transitioner, the effect type table, the drop-item " + "shading) need them, and without them those artifacts are " + "skipped with the reason") + x.add_argument("--player-declarations", metavar="FILE", + help="the same build's declaration dump; names the effect types in " + "effect-resources/ (without it the names are null)") + x.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + x.add_argument("--master-cache", help="where fetched tables are cached") + x.add_argument("--fixture-meshes", action="store_true", + help="also write furniture geometry (fixture-models/), one " + "glTF binary per package; without it that pass is " + "skipped and the passes that need the geometry report " + "it as a missing dependency") + x.add_argument("--fixture-particles", action="store_true", + help="also write furniture particle emitters " + "(fixture-particles/), one document per package " + "across the fixture-interface, cutscene-timeline and " + "fixture-timeline families; without it that pass is " + "skipped. Playable output requires --shader-compiler; " + "without it only source data is saved, not playable assets") + x.add_argument("--shader-compiler", metavar="PATH", + help="pinned offline source shader compiler required for playable " + "furniture particles; omitting it saves source-only data") + x.add_argument("--builtin-resources", action="append", default=[], + metavar="PATH", + help="the engine's own built-in resource container, or a " + "directory holding it; some particle-system renderers " + "and some phenomena emitters draw a copy of a " + "built-in primitive mesh and no package ships those. " + "Repeatable. Without it those mesh slots stay listed " + "as unresolved in fixture-particles/ and in the " + "phenomena index, exactly as before") + x.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + x.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") + x.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") + c = sub.add_parser("characters", help="extract one character bundle") + c.add_argument("--bundle", required=True); c.add_argument("--out-dir", required=True); c.add_argument("--name", required=True) + c.add_argument("--aux-bundle", action="append", default=[]); c.add_argument("--motion-bundle"); c.add_argument("--clips"); c.add_argument("--unit-id", type=int); c.add_argument("--facial-tables"); c.add_argument("--atlas-cell", default="512x256"); c.add_argument("--no-cloth", action="store_true") + m = sub.add_parser("motion-library", help="export the shared humanoid motion library") + m.add_argument("--reference-bundle", required=True); m.add_argument("--motion-bundle", required=True); m.add_argument("--aux-bundle", action="append", default=[]); m.add_argument("--clips"); m.add_argument("--out-dir", required=True); m.add_argument("--name", default="motion-library") + m.add_argument("--controller-bundle", action="append", default=None, + help="character package whose AnimatorController selects the clips " + "(by serialized identity); repeat per package. Without it the " + "first clip of each name is kept and duplicates are listed") + m.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="rotation key storage (default: snorm16); float32 disables lossy encoding") + t = sub.add_parser("facial-tables", help="dump facial tables") + t.add_argument("--bundle", required=True); t.add_argument("--out", required=True) + n = sub.add_parser("alone-actions", help="dump per-character performance data (motion x facial x timing)") + n.add_argument("--bundle", required=True); n.add_argument("--out", required=True) + g = sub.add_parser("registry", help="build the character registry from master tables") + g.add_argument("--master", help="directory of caller-supplied master tables") + g.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + g.add_argument("--master-cache", help="where fetched tables are cached") + g.add_argument("--manifest", required=True, help="manifest naming the character bundles") + g.add_argument("--motion-index", help="shared motion library index, to check motion names") + g.add_argument("--out", required=True) + + k = sub.add_parser("talks", help="extract ordinary solo and multi-character talk scripts") + k.add_argument("--master", help="directory of caller-supplied master tables") + k.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + k.add_argument("--master-cache", help="where fetched tables are cached") + k.add_argument("--bundle", required=True, help="talk scenario bundle") + k.add_argument("--lib", help="decrypted lib package carrying the constant " + "tables the scripts name; without it every " + "Table.key token stays a source token") + k.add_argument("--out", required=True) + + tc = sub.add_parser("text-catalog", help="export versioned display-text identities for translation") + tc.add_argument("--corpus", action="append", required=True, help="extracted talk document; repeatable") + tc.add_argument("--region", required=True, choices=("jp", "cn", "tw", "en", "kr")) + tc.add_argument("--source-revision", required=True, help="explicit source resource revision") + tc.add_argument("--previous", help="previous same-region text catalogue; preserves stable IDs") + tc.add_argument("--out", required=True) + ot = sub.add_parser("official-text", help="match official display text from another source region") + ot.add_argument("--source", required=True, help="source text catalogue") + ot.add_argument("--donor", required=True, help="official regional text catalogue") + ot.add_argument("--locale", required=True, choices=("zh-CN", "zh-TW", "en-US", "ko-KR")) + ot.add_argument("--out", required=True) + + tw = sub.add_parser("tweets", help="extract tweet texts and per-character after-edit pools") + tw.add_argument("--master", help="directory of caller-supplied master tables") + tw.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + tw.add_argument("--master-cache", help="where fetched tables are cached") + tw.add_argument("--out", required=True) + + tt = sub.add_parser("tweet-tables", + help="extract the tweet selection tables (greetings, " + "greeting conditions, site entries, talk pre-actions)") + tt.add_argument("--master", help="directory of caller-supplied master tables") + tt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + tt.add_argument("--master-cache", help="where fetched tables are cached") + tt.add_argument("--out", required=True) + + sg = sub.add_parser("site-groups", + help="extract the mysekai site-group table " + "(siteGroupId -> the site ids it holds)") + sg.add_argument("--master", help="directory of caller-supplied master tables") + sg.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + sg.add_argument("--master-cache", help="where fetched tables are cached") + sg.add_argument("--out", required=True) + + bp = sub.add_parser("birthday-parties", + help="extract the birthday-party campaign table the " + "site-map festival gate reads (startAt/closedAt windows)") + bp.add_argument("--master", help="directory of caller-supplied master tables") + bp.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bp.add_argument("--master-cache", help="where fetched tables are cached") + bp.add_argument("--out", required=True) + + bd = sub.add_parser("birthday-party-delivery", + help="extract the birthday-party delivery tables the " + "delivery site reads (rewards, point bonuses, total rewards)") + bd.add_argument("--master", help="directory of caller-supplied master tables") + bd.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bd.add_argument("--master-cache", help="where fetched tables are cached") + bd.add_argument("--out", required=True) + + gv = sub.add_parser("gate-visit-policy", + help="extract the gate visit tables as policy data for the " + "server mock (not a client character input)") + gv.add_argument("--master", help="directory of caller-supplied master tables") + gv.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + gv.add_argument("--master-cache", help="where fetched tables are cached") + gv.add_argument("--out", required=True) + + tm = sub.add_parser("time-manager", + help="extract the project TimeManager settings (fixed step, " + "maximum allowed timestep, time scale, particle step) " + "from an APK or a globalgamemanagers file") + tm.add_argument("--source", required=True, + help="an APK, or a player-data globalgamemanagers file") + tm.add_argument("--out", required=True) + + cc = sub.add_parser("client-config", + help="extract the ClientConfig deliverable panel " + "(the four typed dictionaries, keyed by id)") + cc.add_argument("--master", help="directory of caller-supplied master tables") + cc.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + cc.add_argument("--master-cache", help="where fetched tables are cached") + cc.add_argument("--out", required=True) + + from core.master_tables import REGISTERED_TABLES + mt = sub.add_parser("master-tables", + help="extract selected key-addressed mysekai master " + "tables, defaulting to blueprints, items, music " + "records and wordings") + mt.add_argument("--master", help="directory of caller-supplied master tables") + mt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + mt.add_argument("--master-cache", help="where fetched tables are cached") + mt.add_argument("--table", dest="tables", action="append", + choices=tuple(REGISTERED_TABLES), + help="registered table to export; repeat to select more " + "(omitting this exports only the original four tables)") + mt.add_argument("--out", required=True, + help="directory the selected documents are written into") + + e = sub.add_parser("emoticons", help="extract overhead-item effect packages") + e.add_argument("--bundle", action="append", required=True); e.add_argument("--out-dir", required=True) + v = sub.add_parser("avatar-parts", help="extract player-appearance packages (skin/decoration/penlight)") + v.add_argument("--bundle", action="append", required=True); v.add_argument("--out-dir", required=True) + ht = sub.add_parser("harvest-tools", help="extract original harvest-tool prefabs and their material data") + ht.add_argument("--bundle", action="append", required=True) + ht.add_argument("--bundle-root", help="directory containing declared shader dependencies") + ht.add_argument("--out-dir", required=True, help="tool directory, normally /avatar/tools") + ptm = sub.add_parser("player-tool-models", + help="extract the avatar's hand-item prefabs and their material data " + "(mysekai/player_tool_model/*)") + ptm.add_argument("--bundle", action="append", required=True) + ptm.add_argument("--bundle-root", help="directory containing declared shader dependencies") + ptm.add_argument("--out-dir", required=True, + help="model directory, normally /avatar/player-tool-models") + er = sub.add_parser("effect-resources", + help="export the effect manager's type table " + "(Resources Mysekai/Effects/EffectResourceData) from player data") + er.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + er.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + er.add_argument("--declarations", + help="the same build's declaration dump, for effect type names") + er.add_argument("--bundle-root", + help="bundle directory to check each row's bundle and prefab against") + er.add_argument("--out", required=True, help="output JSON path") + dis = sub.add_parser("drop-item-shade", + help="export the drop-item shading settings of the graphics " + "configuration (Resources Graphics/Data/MysekaiGraphicsConfig)") + dis.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + dis.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + dis.add_argument("--out", required=True, help="output JSON path") + sts = sub.add_parser("stencil-shadow", + help="export the character stencil-shadow colour and the material " + "rendering settings of the graphics configuration") + sts.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + sts.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + sts.add_argument("--out", required=True, help="output JSON path") + lgr = sub.add_parser("layout-grid", + help="export the layout editor's placement grid: its settings, " + "materials, shader programs and colour profiles " + "(Resources Data/MysekaiGridSettings and " + "Data/GridColorProfileTable)") + lgr.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + lgr.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + lgr.add_argument("--out-dir", required=True, + help="output directory, normally /layout-grid") + rpf = sub.add_parser("resource-prefab", + help="extract one prefab the player data ships in Resources, by load name") + rpf.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + rpf.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + rpf.add_argument("--load-name", required=True, help="the Resources.Load path") + rpf.add_argument("--out-dir", required=True, + help="directory receiving /.{json,glb} and textures") + rpf.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", + help="the engine's built-in resource file (unity default resources, " + "unity_builtin_extra) or a directory holding them; a container " + "build keeps them beside data.unity3d; repeat for each") + rpr = sub.add_parser("resource-programs", + help="write the compiled programs of every shader a Resources " + "prefab's materials use (the program library layout)") + rpr.add_argument("--player-data", required=True, action="append", + help=PLAYER_DATA_HELP) + rpr.add_argument("--types", required=True, + help="managed assembly stubs of the same build (DummyDll directory)") + rpr.add_argument("--load-name", required=True, help="the Resources.Load path") + rpr.add_argument("--out-dir", required=True, + help="directory receiving /shaders/ (the resource-prefab out-dir)") + rpr.add_argument("--shader-compiler", + help="offline shader compiler executable; without it the programs are " + "written as source only (no conversions)") + fp = sub.add_parser("fixture-particles", + help="write particle source-schema documents (the fixture-particles " + "layout) for explicitly named bundles") + fp.add_argument("--bundle", action="append", required=True) + fp.add_argument("--bundle-root", help="directory the bundles' dependencies are read from") + fp.add_argument("--domain", action="append", + help="routed domain to extract; repeat; default the three furniture domains") + fp.add_argument("--shader-compiler", metavar="PATH", + help="pinned offline shader compiler; without it programs stay source-only") + fp.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", + help="the engine's built-in resource container (or its directory), " + "for renderers that point at a built-in mesh") + fp.add_argument("--out-dir", required=True) + pp = sub.add_parser("prefab-package", + help="extract one named prefab of a bundle with the site package reader") + pp.add_argument("--bundle", required=True, help="the bundle file") + pp.add_argument("--prefab", required=True, + help="the prefab's container basename, e.g. fieldcamera.prefab") + pp.add_argument("--bundle-root", help="directory the bundle's dependencies are read from") + pp.add_argument("--out-dir", required=True, + help="directory receiving /.{json,glb} and textures") + sae = sub.add_parser("site-action-effects", + help="extract the site-move and harvest-action effect prefabs " + "(mysekai/effect/site/harvest/action/*)") + sae.add_argument("--bundle", action="append", required=True) + sae.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + sae.add_argument("--out-dir", required=True, + help="effect directory, normally /site-action-effects") + she = sub.add_parser("site-home-effects", + help="extract the home-site effect prefabs, the craft effect among " + "them (mysekai/effect/site/home/*)") + she.add_argument("--bundle", action="append", required=True) + she.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + she.add_argument("--out-dir", required=True, + help="effect directory, normally /site-home-effects") + fpe = sub.add_parser("fixture-put-effects", + help="extract the furniture placement effect prefab " + "(mysekai/effect/fixture/fx_fixture_put_01)") + fpe.add_argument("--bundle", action="append", required=True) + fpe.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + fpe.add_argument("--out-dir", required=True, + help="effect directory, normally /fixture-put-effects") + sep = sub.add_parser("site-edit-pick", + help="export tagged editor raycast colliders in authored Unity coordinates") + sep.add_argument("--bundle", required=True, + help="one mysekai__site__field__ source bundle") + sep.add_argument("--apk", action="append", required=True, + help="matching client APK containing TagManager; repeat for split APKs") + sep.add_argument("--out", required=True, + help="output sidecar, normally /site/edit-pick/.json") + ssw = sub.add_parser("site-sound-wave", + help="resolve MysekaiSiteView._soundWaveMeshFilter (the footstep " + "surface) of site scene packages: node, mesh and vertex colours") + ssw.add_argument("--bundle", action="append", required=True, + help="a mysekai__site__field__ package; repeat") + ssw.add_argument("--bundle-root", help="directory the scene packages' dependencies are read from") + ssw.add_argument("--out", required=True, help="output JSON path") + sce = sub.add_parser("site-common-action-effects", + help="extract the player's foot effect prefabs " + "(mysekai/effect/site/common/action/*)") + sce.add_argument("--bundle", action="append", required=True) + sce.add_argument("--bundle-root", + help="directory containing declared effect, texture and shader dependencies") + sce.add_argument("--out-dir", required=True, + help="effect directory, normally /site-common-action-effects") + sho = sub.add_parser("site-harvest-object-effects", + help="extract the harvest-object hit, delete and stay effect prefabs " + "(mysekai/effect/site/harvest/object//*)") + sho.add_argument("--bundle", action="append", required=True) + sho.add_argument("--bundle-root", + help="directory containing declared effect, mesh, texture and shader dependencies") + sho.add_argument("--out-dir", required=True, + help="effect directory, normally /site-harvest-object-effects") + pa = sub.add_parser("player-avatar", + help="export the player avatar composite (audience model + skeleton + motion clips) as one glb") + pa.add_argument("--bundle", action="append", required=True, + help="the avatar model bundle first, then every motion bundle whose clips " + "are to be resolved against it (the order export_player_avatar takes); " + "repeat --bundle per package") + pa.add_argument("--out-dir", required=True) + pa.add_argument("--name", default="mysekai__player_avatar", + help="output basename (glb + .index.json + .motion-manifest.json)") + w = sub.add_parser("phenomena", help="extract weather (phenomena) environment packages") + w.add_argument("--bundle", action="append", required=True, + help="an environment package, or the shared phenomena thumbnail package") + w.add_argument("--bundle-root", + help="directory the packages a bundle depends on are read from") + w.add_argument("--builtin-resources", action="append", default=[], + metavar="PATH", + help="the engine's own built-in resource container, or a " + "directory holding it; some emitters draw copies of a " + "built-in primitive and no package ships those. Repeatable. " + "Without it those meshes stay listed under `unsupported`, " + "exactly as before") + w.add_argument("--shader-compiler", metavar="PATH", + help="moly-shader-bridge executable; without it source programs " + "are preserved and their conversion is explicitly unavailable") + w.add_argument("--site-bundle", action="append", default=None, metavar="PATH", + help="a site scene package whose site prefab colliders go into " + "collision.json; repeatable. Without it every scene package " + "under --bundle-root is used") + w.add_argument("--out-dir", required=True) + w.add_argument("--master", help="directory of caller-supplied master tables") + w.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + w.add_argument("--master-cache", help="where fetched tables are cached") + w.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it; " + "without it PATH is searched and the audio " + "entry says what is missing") + w.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + a = sub.add_parser("mysekai-audio", + help="extract the rest of the sound corpus: talk voices " + "named by talks.json, plus the remaining se, bgm " + "and part-voice packages") + a.add_argument("--talks", required=True, + help="a talk corpus in either shape this repo's extractors " + "write (talks grouped under units, or a flat talks " + "list); its voice cues are the talk-voice denominator, " + "and only those cues are decoded") + a.add_argument("--manifest", required=True, + help="asset bundle manifest giving the family denominators") + a.add_argument("--bundle-root", required=True, + help="directory holding the decrypted sound packages") + a.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "the products and whose loop.json is merged into") + a.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + a.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + ca = sub.add_parser("cutscene-audio", + help="decode the cues a cut-scene's own timeline names: its " + "voice clips from the cut-scene's birthday voice bank, " + "its music and effect clips from the packages they name") + ca.add_argument("--clips", required=True, action="append", + help="a cut-scene clip document (cutscene-timeline/clips/" + ".json); repeat for each cut-scene") + ca.add_argument("--cut-scenes", required=True, + help="the cut-scene master table (mysekaiCutScenes.json), " + "which routes voice clips to their voice bank") + ca.add_argument("--manifest", required=True, + help="asset bundle manifest the named packages are checked against") + ca.add_argument("--bundle-root", required=True, + help="directory holding the decrypted sound packages") + ca.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "the products and whose loop.json and corpus.json are merged into") + ca.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + ca.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " + "copy of each decoded sound") + sq = sub.add_parser("audio-sequences", + help="attach the cues' sequence structure (type, repeat, " + "per-track delay, track -> stream) to an existing " + "loop sidecar, reading the archives again from the " + "decrypted tree; writes a copy, never in place") + sq.add_argument("--loop", required=True, help="the loop sidecar to read") + sq.add_argument("--bundle-root", required=True, action="append", + help="directory holding the decrypted sound packages; repeat " + "to give further trees, tried in order per entry") + sq.add_argument("--out", required=True, + help="where the annotated copy is written (must not be --loop)") + ra = sub.add_parser("record-audio", + help="read the waveform facts (length, loop region or none, " + "cue commands) of every package a music record can " + "play; the audio itself is not written") + ra.add_argument("--master", help="directory of caller-supplied master tables") + ra.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + ra.add_argument("--master-cache", help="where fetched tables are cached") + ra.add_argument("--bundle-root", required=True, action="append", + help="directory holding decrypted bundles; repeat to give further trees") + ra.add_argument("--bundle-version", required=True, + help="the asset version the bundles were downloaded at (written on every row)") + ra.add_argument("--only", choices=("music", "music_sound_track"), + help="read only the packages of this record track type") + ra.add_argument("--vgmstream", help="path to the external audio decoder " + "(vgmstream-cli), or the directory holding it") + ra.add_argument("--out", required=True, help="the document to write") + rs = sub.add_parser("record-audio-storage", + help="add each row's storage mirror audio file: sha256 and size of " + "the downloaded bytes and the listing's version") + rs.add_argument("input", help="a record-audio document") + rs.add_argument("--server", required=True, choices=("jp", "cn")) + rs.add_argument("--workers", type=int, default=4) + rs.add_argument("--out", required=True) + rm = sub.add_parser("record-audio-merge", + help="combine record-audio documents of separate runs") + rm.add_argument("inputs", nargs="+") + rm.add_argument("--out", required=True) + rc = sub.add_parser("record-audio-carry", + help="carry a read record-audio document's facts to another " + "region's packages whose bundles have the same crc and size") + rc.add_argument("--master", help="directory of the target region's master tables") + rc.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + rc.add_argument("--master-cache", help="where fetched tables are cached") + rc.add_argument("--manifest", required=True, + help="the target region's package list (AssetBundleInfoNew)") + rc.add_argument("--source", required=True, + help="a record-audio document with storage attached") + rc.add_argument("--source-manifest", required=True, + help="the package list of the bundles the source document was read from") + rc.add_argument("--region", required=True, choices=("jp", "cn"), + help="the target region, written on the document") + rc.add_argument("--out", required=True) + rf = sub.add_parser("record-audio-fetch", + help="download and check (size, uncompressed crc) the bundles of " + "the packages a record-audio document lists as absent") + rf.add_argument("document", help="a record-audio document (its absent list is fetched)") + rf.add_argument("--manifest", required=True, + help="the package list (AssetBundleInfoNew) the bundles are fetched by") + rf.add_argument("--asset-base-url", required=True, + help="the prefix before each entry's downloadPath") + rf.add_argument("--workers", type=int, default=4) + rf.add_argument("--out", required=True, help="workspace; bundles land in /decrypted") + ac = sub.add_parser("audio-categories", + help="write the sound configuration's category table (name, " + "index, volume) the runtime scales cues by") + ac.add_argument("--configuration", required=True, + help="the sound configuration (.acf), or the application " + "package (zip) that holds assets/*.acf") + ac.add_argument("--out", required=True, + help="where categories.json is written (the runtime reads " + "phenomena/audio/categories.json)") + r = sub.add_parser("partvoice-routes", + help="build the speaker-to-partvoice-package routing " + "table from master tables and the roster") + r.add_argument("--master", help="directory of caller-supplied master tables") + r.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + r.add_argument("--master-cache", help="where fetched tables are cached") + r.add_argument("--roster", required=True, + help="characters.json, the registry artifact; its " + "characters are the participants' universe and order") + r.add_argument("--out-dir", required=True, + help="the phenomena output directory whose audio/ receives " + "partvoice.json") + u = sub.add_parser("ui-action-icons", + help="export the standalone action-button icon bundles " + "as one PNG per bundle plus a manifest") + u.add_argument("--bundle", action="append", required=True, + help="an action-icon bundle; repeat for as many as wanted") + u.add_argument("--out-dir", required=True) + portraits = sub.add_parser("browser-assets", + help="export source character portraits for the browser catalog") + portraits.add_argument("--player-data", required=True, + help="the APK player data containing CharacterSDIconAtlas") + portraits.add_argument("--asset-root", required=True, + help="runtime resource root containing characters.json") + portraits.add_argument("--source-region", required=True, choices=("cn", "jp")) + portraits.add_argument("--source-game-version", required=True, + help="game version of the supplied UI player data") + presentation = sub.add_parser("fixture-presentation", + help="resolve model, wallpaper, floor and custom furniture source routes") + presentation.add_argument("--asset-root", required=True) + presentation.add_argument("--master", required=True) + presentation.add_argument("--bundles", required=True) + validate = sub.add_parser("validate-runtime", help="verify referenced resources and exact audio cue coverage") + validate.add_argument("--asset-root", required=True) + validate.add_argument("--verify-actor-hashes", action="store_true") + fixture_animations = sub.add_parser("fixture-animations", + help="refresh source-bound fixture clips and their authored time domain") + fixture_animations.add_argument("--asset-root", required=True) + fixture_animations.add_argument("--bundles", required=True) + fixture_animations.add_argument("--package", action="append") + fixture_animations.add_argument("--plan", action="store_true") + capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") + capabilities.add_argument("--asset-root", required=True) + fonts = sub.add_parser("text-fonts", help="prepare an independent display-font catalogue") + fonts.add_argument("--base-font", action="append", required=True) + fonts.add_argument("--font", action="append", required=True) + fonts.add_argument("--license", required=True) + fonts.add_argument("--out-dir", required=True) + tutorial = sub.add_parser("tutorial-placements", help="export source NPC tutorial placement poses") + tutorial.add_argument("--apk", action="append", required=True) + tutorial.add_argument("--client-version", required=True) + tutorial.add_argument("--out-dir", required=True) + tutorial.add_argument("--processor-input", help="optional source-qualified canonical intro pose configuration") + lay = sub.add_parser("ui-layout", + help="extract the RectTransform hierarchy of the UI " + "prefabs (screen layers, dialogs) the player data " + "builds, plus a census of the UI package families") + lay.add_argument("--region", choices=("cn", "jp"), default="cn", + help="cn: one player data file whose resources.assets holds " + "every prefab; jp: the base APK and the asset pack, " + "whose Resources index points at one serialized file " + "per prefab") + lay.add_argument("--player-data", default=None, + help="cn: the APK player data file; every screen and dialog " + "prefab lives in its resources.assets (Resources.Load " + "fetches them, so no bundle name routes to them)") + lay.add_argument("--apk", action="append", default=None, + help="jp: an APK whose assets/bin/Data members are player " + "files; give the base APK and the asset pack") + lay.add_argument("--client-version", default=None, + help="jp: the client version the manifest and every document " + "name (ui/jp_decoders.py names the supported ones; each " + "object's chain comes from its serialized type hash); cn: " + "the client version the palette and tween defaults name") + lay.add_argument("--master", default=None, + help="jp: directory of master tables holding wordings.json") + lay.add_argument("--prefab", action="append", default=None, + help="a prefab name to extract (repeatable); a targeted " + "run writes those documents and omits the broad census") + lay.add_argument("--out-dir", required=True, + help="the output directory (ui-layout/) that receives one " + "document per prefab plus census.json") + lay.add_argument("--bundles", default=None, + help="optional decrypted-package directory; named so the " + "census also reports what the per-screen download " + "packages contain (textures and effects, not layout)") + lay.add_argument("--font-bundles", default=None, + help="jp: decrypted-package directory holding font/common " + "(jp download --root font/common); defaults to --bundles") + lay.add_argument("--bundle-manifest", default=None, + help="optional AssetBundleInfoNew.json; supplies the " + "isBuiltin split and declared sizes for the census") + words = sub.add_parser("ui-wordings", + help="write the runtime wording dictionary of a container build: " + "the built-in wording CSV of its language, then the master " + "wordings over it (the document a shared root reads as its " + "wordings.json)") + words.add_argument("--player-data", required=True, + help="the APK player data file whose Resources index holds the wording CSV") + words.add_argument("--master", required=True, + help="directory of master tables holding wordings.json") + words.add_argument("--client-version", required=True, + help="the client version the document's source block names") + words.add_argument("--out", required=True, help="the document path") + reactions = sub.add_parser("fixture-reactions", + help="export the native furniture reaction-character groups") + reactions.add_argument("--bundle", required=True, + help="decrypted mysekai__system__fixture_reaction_data bundle") + reactions.add_argument("--out", required=True, + help="output fixture-reactions.json path") + bem = sub.add_parser("birthday-end-motions", + help="export the waving motion the birthday party " + "plays as it closes (row per character unit)") + bem.add_argument("--bundle", required=True, + help="decrypted mysekai__birthday__birthday_end_character_motion_data_list bundle") + bem.add_argument("--out", required=True, + help="output birthday-end-character-motions.json path") + bpt = sub.add_parser("birthday-party-talks", + help="attach the birthday-party scripts one master " + "table names to an existing talks.json") + bpt.add_argument("--master", help="directory of caller-supplied master tables") + bpt.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + bpt.add_argument("--master-cache", help="where fetched tables are cached") + bpt.add_argument("--bundle", required=True, help="talk scenario bundle") + bpt.add_argument("--lib", help="decrypted lib package carrying the constant " + "tables the scripts name") + bpt.add_argument("--talks", required=True, + help="an existing talks.json whose birthdayParty block is " + "computed from this master partition") + bpt.add_argument("--out", required=True) + f = sub.add_parser("fixture-master-slice", + help="slice mysekaiFixtures down to the columns the " + "runtime reads (button gating and footprint)") + f.add_argument("--master", help="directory of caller-supplied master tables") + f.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + f.add_argument("--master-cache", help="where fetched tables are cached") + f.add_argument("--out", required=True) + q = sub.add_parser("site", help="extract the site (place) asset packages") + q.add_argument("--bundle", action="append", required=True, + help="a package under the site path; repeat for as many as wanted") + q.add_argument("--bundle-root", + help="directory the packages a bundle depends on are read from") + q.add_argument("--out-dir", required=True) + q.add_argument("--master", help="directory of caller-supplied master tables") + q.add_argument("--master-url", nargs="?", const="", default=None, + help="base URL to append
.json to; no value uses the public default base") + q.add_argument("--master-cache", help="where fetched tables are cached") + ua = sub.add_parser("ui-audio", help="extract embedded UI cue sheets with the shared audio decoder") + ua.add_argument("--apk", required=True) + ua.add_argument("--out", required=True) + ua.add_argument("--vgmstream") + ua.add_argument("--ffmpeg") + sls = sub.add_parser("sitemap-screen-layer", + help="write site/sitemap/screen_layer/screen_layer.json from APK " + "members (player data without resources.assets)") + sls.add_argument("--apk", action="append", required=True, + help="an APK whose assets/bin/Data members are player files; " + "give the base APK and the asset pack") + sls.add_argument("--client-version", required=True, + help="the client version the field chains are for") + sls.add_argument("--out-dir", required=True, + help="runtime root; the document lands at " + "site/sitemap/screen_layer/screen_layer.json under it") + mth = sub.add_parser("material-thumbnails", + help="write material-thumbnails.json and textures/material{id}.png " + "from the thumbnail/material package, one per materials row") + mth.add_argument("--bundle", required=True, help="the decrypted thumbnail/material package") + mth.add_argument("--master", required=True, + help="directory of master tables holding materials.json") + mth.add_argument("--out-dir", required=True, + help="output directory (material-thumbnails/ of a runtime root)") + hon = sub.add_parser("honor-images", + help="write honor/index.json and the textures of the honor bundles " + "the birthday-party delivery rewards' honors draw") + hon.add_argument("--master", required=True, + help="directory of master tables (the delivery total rewards, resource boxes, honors, honor groups)") + hon.add_argument("--bundles", required=True, help="directory of decrypted honor and honor_frame packages") + hon.add_argument("--out-dir", required=True, help="runtime root; images land under honor/ and honor_frame/") + ipv = sub.add_parser("item-icons", + help="write index.json and textures///.png from " + "the mysekai item_preview and thumbnail material/item/tool packages, " + "keyed by the package path") + ipv.add_argument("--decrypted", required=True, + help="directory holding the decrypted mysekai item icon packages") + ipv.add_argument("--root", default=None, + help="runtime root whose mysekai-materials/items/fixtures/tools documents " + "give the coverage report") + ipv.add_argument("--out-dir", required=True, + help="output directory (mysekai-item-icons/ of a runtime root)") + a = sub.add_parser("fetch-apk", help="discover, download, or inspect an Android APK") + a.add_argument("--endpoint", default=None); a.add_argument("--timeout", type=float, default=30.0); a.add_argument("--retries", type=int, default=3) + asub = a.add_subparsers(dest="apk_command", required=True) + asub.add_parser("latest") + ad = asub.add_parser("download"); ad.add_argument("destination"); ad.add_argument("--url"); ad.add_argument("--sha256") + ai = asub.add_parser("inspect"); ai.add_argument("apk") + rp = sub.add_parser( + "player-render-pipeline", + help="extract the player-shipped MYSEKAI renderer pipeline and feature settings", + ) + rp.add_argument( + "--player-data", required=True, + help="extracted assets/bin/Data root containing globalgamemanagers", + ) + rp.add_argument( + "--types", required=True, + help="decoded managed assemblies for the same player build", + ) + rp.add_argument( + "--asset-pack", required=True, + help="APK/zip containing serialized resource files referenced by ResourceManager", + ) + rp.add_argument("--game-version", help="caller-known game version recorded as provenance") + rp.add_argument("--out", required=True, help="output JSON evidence path") + fal = sub.add_parser("fixture-actors", help="export all source-qualified character furniture animations") + fal.add_argument("--assets", required=True); fal.add_argument("--master", required=True) + fal.add_argument("--bundles", required=True); fal.add_argument("--workers", type=int, default=2) + fal.add_argument("--plan", action="store_true", help="write and summarize the source dependency plan without exporting") + fal.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="rotation key storage (default: snorm16); float32 disables lossy encoding") + fal.add_argument("--relation", action="append", choices=("fixture", "cut_scene"), default=None, + help="export only these relations' libraries (repeatable; default: all)") + ptc = sub.add_parser("player-timeline-clips", + help="export each player fixture timeline's CharacterAnimator clips against the " + "player avatar rig, one clip file per timeline package") + ptc.add_argument("--assets", required=True, + help="asset root holding fixture-timeline/ and the player timeline rows") + ptc.add_argument("--bundles", required=True, + help="directory of decrypted bundles (the avatar model and the fixture models)") + ptc.add_argument("--plan", action="store_true", help="summarize the clip plan without exporting") + stl = sub.add_parser("site-timelines", + help="export a site package's timeline tables, its step items (each prefab its " + "own glb with its prop clips) and its scene directors' rigs") + stl.add_argument("--assets", required=True, help="asset root receiving site-timeline/") + stl.add_argument("--bundles", required=True, + help="directory of decrypted bundles (the site packages and their dependencies)") + stl.add_argument("--package", action="append", required=True, + help="site package to export, e.g. mysekai__site__field__festivalgarden; repeatable") + fc = sub.add_parser("fixture-coverage", help="join every fixture master ID to extracted coverage") + fc.add_argument("--data-root", default=DEFAULT_DATA_ROOT); fc.add_argument("--out") + fc.add_argument("--region"); fc.add_argument("--source-version"); fc.add_argument("--json", action="store_true") + jp = sub.add_parser("jp", help="discover and download Japanese-server original bundles") + jsub = jp.add_subparsers(dest="jp_command", required=True) + jd = jsub.add_parser("discover", help="compare bootstrap metadata with the live official version list") + jd.add_argument("--bootstrap-url", default=None); jd.add_argument("--timeout", type=float, default=30.0) + jd.add_argument("--retries", type=int, default=3) + jm = jsub.add_parser("manifest", help="create a versioned JP bundle descriptor") + jm.add_argument("--app-version"); jm.add_argument("--app-hash") + jm.add_argument("--asset-version"); jm.add_argument("--asset-hash") + jm.add_argument("--versions-url", default=None) + jm.add_argument("--guest", action="store_true", help="explicitly create a guest only when public asset identity is unavailable") + jm.add_argument("--session", help="private guest-session JSON path; required with --guest") + jm.add_argument("--out", required=True); jm.add_argument("--timeout", type=float, default=30.0) + jm.add_argument("--retries", type=int, default=3) + jm.add_argument("--json", action="store_true", help="print the full descriptor as well as writing --out") + jpl = jsub.add_parser("plan", help="dependency-closed dry run from an online or offline descriptor") + jpl.add_argument("--manifest", required=True); jpl.add_argument("--family", action="append", default=[]) + jpl.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") + jpl.add_argument("--json", action="store_true", help="print root and bundle lists, not only counts") + jrt = jsub.add_parser("prepare", help="assemble a region-qualified Moly runtime in a separate staging directory") + jrt.add_argument("--manifest", required=True); jrt.add_argument("--workspace", required=True) + jrt.add_argument("--out", required=True); jrt.add_argument("--ui-from", required=True) + jrt.add_argument("--ui-jp-from", required=True, + help="JP UI extraction (ui-layout --region jp); its documents replace their donor copies") + jrt.add_argument("--vgmstream"); jrt.add_argument("--ffmpeg") + jrt.add_argument("--master-from", help="completed source-pinned master directory; defaults to Haruki when available") + jrt.add_argument("--actor-workers", type=int, default=2) + jrt.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, + help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") + jms = jsub.add_parser("master", help="download an immutable, bundle-matched JP master snapshot") + jms.add_argument("--manifest", required=True); jms.add_argument("--out", required=True) + jms.add_argument("--revision", default="main"); jms.add_argument("--workers", type=int, default=6) + jms.add_argument("--provider", choices=("haruki", "sekai-world"), default="haruki") + jms.add_argument("--all-tables", action="store_true", help="fetch all master tables, not only Moly dependencies") + jdl = jsub.add_parser("download", help="download and verify original JP bundles without publishing an extracted release") + jdl.add_argument("--manifest", required=True); jdl.add_argument("--out") + jdl.add_argument("--workers", type=int, default=8); jdl.add_argument("--retries", type=int, default=4) + jdl.add_argument("--root", action="append", default=[]) + jdl.add_argument("--family", action="append", default=[]) + jpu = jsub.add_parser("pull", help="download, decrypt and extract a JP descriptor") + jpu.add_argument("--manifest", required=True); jpu.add_argument("--out") + jpu.add_argument("--extract-out", default="local-data-jp") + jpu.add_argument("--workers", type=int, default=4); jpu.add_argument("--retries", type=int, default=4) + jpu.add_argument("--master"); jpu.add_argument("--master-cache") + jpu.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") + s = sub.add_parser("shader", help="what a package's shaders declare and contain") + s.add_argument("package", help="one AssetBundle file") + s.add_argument("--platform", type=int, default=None, + help="count only this ShaderCompilerPlatform (9 is GLSL text)") + s.add_argument("--out", help="write the full census here as JSON") + s.add_argument("--programs", metavar="DIR", + help="write each program body here, named by shader, platform and record") + s.add_argument("--json", action="store_true", help="print the counts as JSON") + + args = ap.parse_args(argv); _configure_unity(args.unity_version) + if args.cmd == "fixture-actors": + from chara.fixture_actor_libraries import build_plan, export_fixture_actor_libraries + if args.plan: + report = build_plan(args.assets, args.master, args.bundles, rotation_encoding=args.rotation_encoding) + print(json.dumps({"summary": report["summary"], "gaps": report["gaps"]}, ensure_ascii=False)) + return 1 if report["gaps"] else 0 + report = export_fixture_actor_libraries(args.assets, args.master, args.bundles, + workers=args.workers, unity_version=args.unity_version or "2022.3.62f3", + rotation_encoding=args.rotation_encoding, relations=args.relation, + progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) + print(json.dumps({key: value for key, value in report.items() if key != "results"}, ensure_ascii=False)) + return 1 if report["failed"] else 0 + if args.cmd == "player-timeline-clips": + from chara.player_timeline_clips import build_plan, export_player_timeline_clips + if args.plan: + plan = build_plan(args.assets) + print(json.dumps({"summary": plan["summary"], "gaps": plan["gaps"]}, ensure_ascii=False)) + return 1 if plan["gaps"] else 0 + report = export_player_timeline_clips(args.assets, args.bundles, + progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) + print(json.dumps({key: value for key, value in report.items() if key != "packages"}, + ensure_ascii=False)) + return 1 if report["gaps"] else 0 + if args.cmd == "site-timelines": + from sites.step_items import export_site_timelines + report = export_site_timelines(args.bundles, args.assets, list(args.package)) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "fixture-coverage": + from fixtures.coverage import write_coverage + path, report = write_coverage(args.data_root, args.out, region=args.region, + source_version=args.source_version) + print(json.dumps(report if args.json else {**report["summary"], "report": str(path)}, ensure_ascii=False)) + return 1 if report["summary"]["runtime"]["failed"] else 0 + if args.cmd == "jp": + from core import jp as jp_assets + try: + if args.jp_command == "discover": + options = {"timeout": args.timeout, "retries": args.retries} + if args.bootstrap_url: options["bootstrap_url"] = args.bootstrap_url + report = jp_assets.discover(**options) + elif args.jp_command == "manifest": + report = jp_assets.fetch_manifest(args.out, app_version=args.app_version, + app_hash=args.app_hash, session_file=args.session, + asset_version=args.asset_version, + asset_hash=args.asset_hash, guest=args.guest, + **({"versions_url": args.versions_url} + if args.versions_url else {}), + timeout=args.timeout, retries=args.retries) + elif args.jp_command == "plan": + report = jp_assets.plan(args.manifest, families=args.family, include=args.root) + elif args.jp_command == "prepare": + from core.runtime_pack import prepare_runtime + report = prepare_runtime(args.manifest, args.workspace, args.out, ui_from=args.ui_from, + ui_jp_from=args.ui_jp_from, + decoder=args.vgmstream, transcoder=args.ffmpeg, + master_from=args.master_from, actor_workers=args.actor_workers, + rotation_encoding=args.rotation_encoding, + progress=lambda label: print(label, file=sys.stderr)) + elif args.jp_command == "master": + from core.jp_master import download_master_snapshot + report = download_master_snapshot(args.manifest, args.out, revision=args.revision, workers=args.workers, provider=args.provider, all_tables=args.all_tables) + report = {"identity": report["identity"], "tableCount": report["tableCount"], "out": args.out} + elif args.jp_command == "download": + from core.jp_download import download_descriptor + report = download_descriptor(args.manifest, out=args.out, workers=args.workers, + retries=args.retries, include=args.root, families=args.family, + progress=lambda row: print(f"{row['status']}: {row['completed']}/{row['requested']} verified, {row['failed']} failed", file=sys.stderr)) + else: + report = jp_assets.pull_descriptor(args.manifest, out=args.out, + extract_out=args.extract_out, workers=args.workers, + retries=args.retries, master=args.master, + master_cache=args.master_cache, + include=args.root) + if args.jp_command == "manifest" and not args.json: + shown = {"source": report["source"], "bundles": len(report["bundles"]), "descriptor": args.out} + elif args.jp_command == "plan" and not args.json: + shown = {"source": report["source"], **report["summary"]} + else: + shown = report + print(json.dumps(shown, ensure_ascii=False)); return 0 + except (jp_assets.JPError, OSError, ValueError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr); return 1 + if args.cmd == "pull": + from core.fetch import pull + source, cache = _master_source(args) + report = pull(args.manifest, args.out, args.asset_base_url, args.workers, args.retries, + master=source, master_cache=cache, extract_out=args.extract_out, + vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) + if args.json: + print(json.dumps(report, ensure_ascii=False)) + else: + print(f"downloaded {report['downloads']}/{report['requiredBundles']} bundles") + # The sound packages are the one part of the set that master rows name, + # so "no audio" has a reason the caller must see without asking for JSON. + audio = report.get("audio") or {} + if audio.get("roots"): + print(f"audio: {len(audio['roots'])} sound packages named by master tables") + elif audio.get("error"): + print(f"audio: no sound package pulled ({audio['error']})") + if audio.get("notInManifest"): + print(f"audio: {len(audio['notInManifest'])} named sound packages " + f"are not in this manifest: " + f"{', '.join(audio['notInManifest'])}") + _print_extract_summary(report["extraction"], args.extract_out or DEFAULT_DATA_ROOT) + return 1 if _extract_failed(report["extraction"]) else 0 + if args.cmd == "shader": + import UnityPy + from shaders import census as shader_census + entries = shader_census.census(UnityPy.load(args.package), + keep_code=bool(args.programs)) + counts = shader_census.totals(entries, args.platform) + if args.out: + from core.jsonio import write_json + write_json(args.out, {"package": args.package, + "shaders": shader_census.without_code(entries)}) + if args.programs: + written = _write_programs(args.programs, entries, args.platform) + counts["bodiesWritten"] = written + if args.json: + print(json.dumps(counts, ensure_ascii=False)) + else: + scope = "all platforms" if args.platform is None else f"platform {args.platform}" + print(f"{counts['shaders']} shaders, {counts['records']} program records, " + f"{counts['uniquePrograms']} distinct programs ({scope})") + if counts["platformErrors"]: + print(f"{counts['platformErrors']} platform blob(s) failed to parse") + for name, records, unique in shader_census.by_shader(entries, args.platform)[:10]: + print(f" {unique:6d} distinct / {records:6d} records {name}") + return 1 if counts["platformErrors"] else 0 + if args.cmd == "ui-audio": + from ui.audio import extract_ui_audio + report = extract_ui_audio(args.apk, args.out, decoder=args.vgmstream, transcoder=args.ffmpeg) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "fetch-apk": + from core import apk + forwarded = [] + if args.endpoint: forwarded += ["--endpoint", args.endpoint] + forwarded += ["--timeout", str(args.timeout), "--retries", str(args.retries), args.apk_command] + if args.apk_command == "download": + forwarded += [args.destination] + if args.url: forwarded += ["--url", args.url] + if args.sha256: forwarded += ["--sha256", args.sha256] + elif args.apk_command == "inspect": + forwarded += [args.apk] + return apk.main(forwarded) + if args.cmd == "player-render-pipeline": + from core.player_render_pipeline import ( + extract_mysekai_render_pipeline, + write_mysekai_render_pipeline, + ) + try: + document = extract_mysekai_render_pipeline( + args.player_data, + args.types, + args.asset_pack, + unity_version=args.unity_version, + game_version=args.game_version, + ) + write_mysekai_render_pipeline(args.out, document) + except (OSError, ValueError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + first = document["rendererSlots"][0]["objectKey"] if document["rendererSlots"] else None + count = len(document["renderers"].get(first, {}).get("features", [])) if first else 0 + print(json.dumps({ + "unityVersion": document["unityVersion"], + "gameVersion": document["gameVersion"], + "rendererSlots": len(document["rendererSlots"]), + "primaryRendererFeatures": count, + "out": args.out, + }, ensure_ascii=False)) + return 0 + if args.cmd == "extract": + from core.extract import extract_manifest + from core.assets.packages import builtin_archive_paths + source, cache = _master_source(args) + report = extract_manifest(args.manifest, args.bundles, args.out, args.unity_version, + master=source, master_cache=cache, + player_data=args.player_data, + player_types=args.player_types, + player_declarations=args.player_declarations, + fixture_meshes=args.fixture_meshes, + fixture_particles=args.fixture_particles, + rotation_encoding=args.rotation_encoding, + shader_compiler=args.shader_compiler, + builtin_resources=builtin_archive_paths( + args.builtin_resources), + vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) + if args.json: + print(json.dumps(report, ensure_ascii=False)) + else: + if report.get("discovery"): + d = report["discovery"] + print(f"discovered {d['selected']} pack bundles under {args.bundles} " + f"(ignored {d['ignored']} unrelated files)") + _print_extract_summary(report, args.out) + return 1 if _extract_failed(report) else 0 + from chara import characters + if args.cmd == "motion-library": + from chara.motion_library import export_motion_library + names = [s for s in args.clips.split(",") if s] if args.clips else None + report = export_motion_library(args.reference_bundle, args.motion_bundle, args.out_dir, args.name, + aux=tuple(args.aux_bundle), names=names, + rotation_encoding=args.rotation_encoding, + controller_bundles=args.controller_bundle) + print(json.dumps(report, ensure_ascii=False)); return 0 + if args.cmd == "registry": + source, cache = _master_source(args) + if not source: + ap.error("registry needs master tables: pass --master or --master-url") + from core.assets.manifest import parse_manifest + from core.assets.router import route + from core.extract import _unit_id + from core.master import Master, MissingTable + from chara.registry import build_registry + names = parse_manifest(args.manifest) + units = sorted({u for n in names if route(n) and route(n).domain == "character" + for u in [_unit_id(n)] if u is not None}) + index = None + if args.motion_index: + with open(args.motion_index, encoding="utf-8") as handle: + index = json.load(handle) + source, cache = _master_source(args) + master = Master(source, cache_dir=cache) + try: + client_configs = master.client_configs() + except MissingTable: + client_configs = None + document = build_registry(master, units, index, client_configs=client_configs) + with open(args.out, "w", encoding="utf-8", newline="\n") as handle: + json.dump(document, handle, ensure_ascii=False, indent=1, allow_nan=False) + handle.write("\n") + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + + if args.cmd == "official-text": + from chara.text_catalog import match_official + from core.jsonio import write_json + with open(args.source, encoding="utf-8") as handle: + source = json.load(handle) + with open(args.donor, encoding="utf-8") as handle: + donor = json.load(handle) + result = match_official(source, donor, locale=args.locale) + write_json(args.out, result) + print(json.dumps({"entries": len(result["entries"]), "issues": len(result["issues"]), + "revision": result["revision"]})) + return 0 + + if args.cmd == "text-catalog": + from chara.text_catalog import export_catalog + print(json.dumps(export_catalog(args.corpus, args.out, region=args.region, + source_revision=args.source_revision, + previous=args.previous), ensure_ascii=False)) + return 0 + + if args.cmd == "talks": + source, cache = _master_source(args) + if not source: + ap.error("talks needs master tables: pass --master or --master-url") + from chara.talks import extract_talks + source, cache = _master_source(args) + print(json.dumps(extract_talks(source, args.bundle, args.out, + master_cache=cache, + lib_bundle=args.lib), ensure_ascii=False)) + return 0 + + if args.cmd == "tweets": + source, cache = _master_source(args) + if not source: + ap.error("tweets needs master tables: pass --master or --master-url") + from chara.tweets import extract_tweets + print(json.dumps(extract_tweets(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "tweet-tables": + source, cache = _master_source(args) + if not source: + ap.error("tweet-tables needs master tables: " + "pass --master or --master-url") + from chara.tweet_tables import extract_tweet_tables + print(json.dumps(extract_tweet_tables(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "site-groups": + source, cache = _master_source(args) + if not source: + ap.error("site-groups needs master tables: " + "pass --master or --master-url") + from sites.site_groups import extract_site_groups + print(json.dumps(extract_site_groups(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "ui-action-icons": + from ui.action_icon import export_action_icons + print(json.dumps(export_action_icons(args.bundle, args.out_dir), + ensure_ascii=False)) + return 0 + + if args.cmd == "browser-assets": + from ui.character_portrait import export_character_portraits + result = export_character_portraits(args.player_data, args.asset_root, + source_region=args.source_region, + source_game_version=args.source_game_version) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 + + if args.cmd == "fixture-presentation": + from fixtures.presentation import extract_custom_geometry, enrich_presentation + extracted = extract_custom_geometry(args.bundles, Path(args.asset_root) / "custom-fixture-models") + if extracted.get("failed"): + print(json.dumps({"customGeometry": extracted}, ensure_ascii=False)) + return 1 + print(json.dumps(enrich_presentation(args.master, args.asset_root), ensure_ascii=False)) + return 0 + + if args.cmd == "validate-runtime": + from core.runtime_validation import validate_runtime + report = validate_runtime(args.asset_root, verify_actor_hashes=args.verify_actor_hashes) + print(json.dumps(report, ensure_ascii=False)) + return 1 if report["missingResources"] else 0 + + if args.cmd == "fixture-animations": + from fixtures.animation_refresh import plan_animation_refresh, refresh_fixture_animations + report = plan_animation_refresh(args.asset_root) if args.plan else refresh_fixture_animations( + args.asset_root, args.bundles, packages=args.package, + progress=lambda row: print(json.dumps(row), file=sys.stderr)) + print(json.dumps(report, ensure_ascii=False)) + return 0 + + if args.cmd == "timeline-capabilities": + from fixtures.timeline_capabilities import build_capabilities + report = build_capabilities(args.asset_root) + print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) + return 0 + + if args.cmd == "text-fonts": + from ui.text_fonts import build_font_catalog + result = build_font_catalog(args.base_font, args.font, args.license, args.out_dir) + print(json.dumps({"revision": result['revision'], "fonts": len(result['fonts'])})) + return 0 + + if args.cmd == "tutorial-placements": + from chara.tutorial import export_placements + result = export_placements(args.apk, args.out_dir, client_version=args.client_version, + processor_input=args.processor_input) + print(json.dumps({"placements": len(result['placements']), + "participants": sum(len(rows) for rows in result['placements'].values())})) + return 0 + + if args.cmd == "ui-layout" and args.region == "jp": + from ui.prefab_layout import extract_layout_jp + if not (args.apk and args.client_version and args.master and args.prefab): + ap.error("ui-layout --region jp needs --apk (base and asset pack), " + "--client-version, --master and at least one --prefab") + result = extract_layout_jp(args.apk, args.out_dir, client_version=args.client_version, + master_dir=args.master, prefabs=args.prefab, + bundles_root=args.bundles, + font_bundles_root=args.font_bundles) + for item in result["written"]: + summary = item["summary"] + print(f"layout: {item['family']}/{item['prefab']} " + f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " + f"texts={summary['textComponents']} " + f"partial={len(summary['partialComponents'])} " + f"unresolved={len(summary['unresolvedPointers'])}") + for item in result["failures"]: + print(f"layout FAILED {item['prefab']}: {item['reason']}") + print(f"manifest: {result['manifestPath']}") + return 0 if not result["failures"] else 1 + + if args.cmd == "ui-layout": + from ui.prefab_layout import extract_layout + if not args.player_data: + ap.error("ui-layout --region cn needs --player-data") + result = extract_layout(args.player_data, args.out_dir, + bundles_root=args.bundles, + bundle_manifest=args.bundle_manifest, + prefabs=args.prefab, + client_version=args.client_version) + for item in result["settings"]: + print(f"settings: {item['document']} from {item['serializedFile']}:{item['pathId']}") + if not result["settings"]: + print("settings: palette.json and tween-settings.json not written " + "(they name the client version: pass --client-version)") + for item in result["written"]: + summary = item["summary"] + print(f"layout: {item['family']}/{item['prefab']} " + f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " + f"texts={summary['textComponents']} " + f"partial={len(summary['partialComponents'])}") + for item in result["failures"]: + print(f"layout FAILED {item['prefab']}: {item['reason']}") + print(f"census: {result['censusPath']}") + return 0 if not result["failures"] else 1 + + if args.cmd == "ui-wordings": + from ui.prefab_layout import extract_container_wordings + result = extract_container_wordings(args.player_data, args.master, args.out, args.client_version) + print(f"wordings: {result['rows']} entries ({result['builtinRows']} built-in, " + f"{result['masterRows']} master rows, {result['masterOverwroteBuiltin']} built-in " + f"overwritten, {result['masterValuesRewritten']} master values rewritten, " + f"{result['builtinRejectedLines']} rejected lines) from " + f"{result['serializedFile']}:{result['pathId']} -> {result['path']}") + return 0 + + if args.cmd == "fixture-reactions": + from fixtures.reactions import extract_fixture_reactions + print(json.dumps(extract_fixture_reactions(args.bundle, args.out), ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-end-motions": + from sites.birthday_end_motions import extract_birthday_end_motions + print(json.dumps(extract_birthday_end_motions(args.bundle, args.out), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-party-talks": + source, cache = _master_source(args) + if not source: + ap.error("birthday-party-talks needs master tables: " + "pass --master or --master-url") + from chara.talks import attach_birthday_talks + print(json.dumps(attach_birthday_talks(source, args.bundle, args.talks, + args.out, master_cache=cache, + lib_bundle=args.lib), + ensure_ascii=False)) + return 0 + + if args.cmd == "fixture-master-slice": + source, cache = _master_source(args) + if not source: + ap.error("fixture-master-slice needs master tables: " + "pass --master or --master-url") + from fixtures.master_slice import export_fixture_master_slice + print(json.dumps(export_fixture_master_slice(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-parties": + source, cache = _master_source(args) + if not source: + ap.error("birthday-parties needs master tables: " + "pass --master or --master-url") + from sites.birthday_parties import extract_birthday_parties + print(json.dumps(extract_birthday_parties(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "birthday-party-delivery": + source, cache = _master_source(args) + if not source: + ap.error("birthday-party-delivery needs master tables: " + "pass --master or --master-url") + from sites.birthday_party_delivery import extract_birthday_party_delivery + print(json.dumps(extract_birthday_party_delivery(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "gate-visit-policy": + source, cache = _master_source(args) + if not source: + ap.error("gate-visit-policy needs master tables: " + "pass --master or --master-url") + from sites.gate_visit_policy import extract_gate_visit_policy + print(json.dumps(extract_gate_visit_policy(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + if args.cmd == "time-manager": + from core.time_manager import extract_time_manager + document = extract_time_manager(args.source, args.out) + print(json.dumps({"source": document["source"], "timeManager": document["timeManager"]}, + ensure_ascii=False)) + return 0 + if args.cmd == "client-config": + source, cache = _master_source(args) + if not source: + ap.error("client-config needs master tables: " + "pass --master or --master-url") + from core.client_config import extract_client_config + print(json.dumps(extract_client_config(source, args.out, master_cache=cache), + ensure_ascii=False)) + return 0 + + if args.cmd == "master-tables": + source, cache = _master_source(args) + if not source: + ap.error("master-tables needs master tables: " + "pass --master or --master-url") + from core.master_tables import extract_master_tables + print(json.dumps(extract_master_tables(source, args.out, + master_cache=cache, tables=args.tables), + ensure_ascii=False)) + return 0 + + if args.cmd == "emoticons": + from chara.emoticons import extract_emoticons + print(json.dumps(extract_emoticons(args.bundle, args.out_dir), ensure_ascii=False)) + return 0 + if args.cmd == "avatar-parts": + from chara.avatar_parts import extract_avatar_parts + print(json.dumps(extract_avatar_parts(args.bundle, args.out_dir), ensure_ascii=False)) + return 0 + if args.cmd == "harvest-tools": + from chara.harvest_tools import extract_harvest_tools + result = extract_harvest_tools(args.bundle, args.out_dir, bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "player-tool-models": + from chara.harvest_tools import extract_player_tool_models + result = extract_player_tool_models(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "effect-resources": + from core.resource_exports import export_effect_resources + print(json.dumps(export_effect_resources( + args.player_data, args.types, args.out, declarations=args.declarations, + bundle_root=args.bundle_root), ensure_ascii=False)) + return 0 + if args.cmd == "layout-grid": + from core.layout_grid import export_layout_grid + print(json.dumps(export_layout_grid(args.player_data, args.types, args.out_dir), + ensure_ascii=False, indent=2)) + return 0 + if args.cmd == "drop-item-shade": + from core.resource_exports import export_drop_item_shade + print(json.dumps(export_drop_item_shade(args.player_data, args.types, args.out), + ensure_ascii=False)) + return 0 + if args.cmd == "stencil-shadow": + from core.resource_exports import export_stencil_shadow + print(json.dumps(export_stencil_shadow(args.player_data, args.types, args.out), + ensure_ascii=False)) + return 0 + if args.cmd == "resource-prefab": + from core.resource_exports import export_resource_prefab + from core.assets.packages import builtin_archive_paths + document = export_resource_prefab(args.player_data, args.types, + args.load_name, args.out_dir, + builtin_archive_paths(args.builtin_resources)) + print(json.dumps({"file": document["file"], "objects": document["objects"], + "roots": [root["name"] for root in document["roots"]], + "particles": len(document["particles"]), + "unsupported": len(document["unsupported"])}, + ensure_ascii=False)) + return 0 + if args.cmd == "resource-programs": + from core.resource_exports import export_resource_programs + document = export_resource_programs(args.player_data, args.types, args.load_name, + args.out_dir, args.shader_compiler) + print(json.dumps({"file": document["file"], + "shaders": [(row["name"], row["state"], len(row["materials"])) + for row in document["shaders"]]}, ensure_ascii=False)) + return 0 + if args.cmd == "fixture-particles": + from core.assets.packages import PackageStore, builtin_archive_paths, require_paths + from fixtures.particles import DOMAINS, extract_from_store + require_paths(args.bundle) + builtins = builtin_archive_paths(args.builtin_resources) + require_paths(builtins, "built-in resource") + store = PackageStore(list(args.bundle) + builtins, args.bundle_root) + result = extract_from_store(store, args.out_dir, + shader_compiler=args.shader_compiler, + domains=tuple(args.domain or DOMAINS)) + print(json.dumps(result, ensure_ascii=False)) + return 0 + if args.cmd == "prefab-package": + from sites.prefab_package import extract_prefab_package + print(json.dumps(extract_prefab_package(args.bundle, args.prefab, args.out_dir, + bundle_root=args.bundle_root), + ensure_ascii=False)) + return 0 + if args.cmd == "site-edit-pick": + from sites.edit_pick import extract_site_edit_pick + print(json.dumps(extract_site_edit_pick(args.bundle, args.apk, args.out), + ensure_ascii=False)) + return 0 + if args.cmd == "site-sound-wave": + from sites.sound_wave import extract_site_sound_wave + result = extract_site_sound_wave(args.bundle, args.out, bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if result["failed"] else 0 + if args.cmd == "site-common-action-effects": + from sites.action_effects import extract_site_common_action_effects + result = extract_site_common_action_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-harvest-object-effects": + from sites.action_effects import extract_site_harvest_object_effects + result = extract_site_harvest_object_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-action-effects": + from sites.action_effects import extract_site_action_effects + result = extract_site_action_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "site-home-effects": + from sites.action_effects import extract_site_home_effects + result = extract_site_home_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "fixture-put-effects": + from sites.action_effects import extract_fixture_put_effects + result = extract_fixture_put_effects(args.bundle, args.out_dir, + bundle_root=args.bundle_root) + print(json.dumps(result, ensure_ascii=False)) + return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 + if args.cmd == "player-avatar": + from chara.player_avatar import export_player_avatar, write_motion_manifest + record = export_player_avatar(args.bundle, args.out_dir, name=args.name) + manifest_path = str(Path(args.out_dir) / f"{args.name}.motion-manifest.json") + write_motion_manifest(record, manifest_path) + print(json.dumps({ + "glb": record["glb"], "index": record["index"], + "motionManifest": manifest_path, + "sourcePackages": record["sourcePackages"], + "playerMesh": record["playerMesh"], + "counts": record["counts"]}, ensure_ascii=False)) + return 0 + if args.cmd == "phenomena": + from core.assets.packages import builtin_archive_paths, require_paths + from phenomena.environments import extract_phenomena + source, cache = _master_source(args) + # --bundle takes paths, not bare package names: PackageStore keys the + # store by basename and UnityPy.load answers a path that is not there + # with an empty environment and no error, so a bare name runs green + # over zero objects. Refuse it here, before any work is done. + require_paths(args.bundle) + require_paths(args.site_bundle) + report = extract_phenomena(args.bundle, args.out_dir, + bundle_root=args.bundle_root, master=source, + master_cache=cache, vgmstream=args.vgmstream, + ffmpeg=args.ffmpeg, + shader_compiler=args.shader_compiler, + extra_archives=builtin_archive_paths( + args.builtin_resources), + site_bundles=args.site_bundle) + print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, + ensure_ascii=False)) + return 0 + if args.cmd == "mysekai-audio": + from phenomena.audio_corpus import extract_audio_corpus + report = extract_audio_corpus(args.talks, args.manifest, + args.bundle_root, args.out_dir, + decoder=args.vgmstream, + transcoder=args.ffmpeg) + print(json.dumps({k: v for k, v in report.items() if k != "audio"}, + ensure_ascii=False)) + return 0 + if args.cmd == "cutscene-audio": + from phenomena.cutscene_audio import extract_cutscene_audio + report = extract_cutscene_audio(args.clips, args.cut_scenes, args.manifest, + args.bundle_root, args.out_dir, + decoder=args.vgmstream, transcoder=args.ffmpeg) + print(json.dumps(report, ensure_ascii=False)) + return 0 + if args.cmd == "audio-sequences": + if Path(args.out).resolve() == Path(args.loop).resolve(): + ap.error("audio-sequences writes a copy: --out must differ from --loop") + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.audio_sequences import annotate_loop + report = annotate_loop(args.loop, args.bundle_root, args.out) + print(json.dumps({"counts": report["counts"], + "structuredByType": report["structuredByType"], + "archiveCuesByReferenceType": report["archiveCuesByReferenceType"], + "archiveSequencesByType": report["archiveSequencesByType"], + "answeredByTree": {k: len(v) for k, v + in report["answeredByTree"].items()}, + "failures": len(report["failures"]), + "disagreements": len(report["disagreements"]), + "sampleDifferences": len(report["sampleDifferences"])}, + ensure_ascii=False)) + return 0 + if args.cmd == "record-audio": + source, cache = _master_source(args) + if not source: + ap.error("record-audio needs master tables: " + "pass --master or --master-url") + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.record_audio import extract_record_audio + document = extract_record_audio(source, args.bundle_root, args.out, + bundle_version=args.bundle_version, + decoder=args.vgmstream, master_cache=cache, + only=args.only) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-storage": + from phenomena.record_audio import attach_storage + document = attach_storage(args.input, args.out, server=args.server, + workers=args.workers) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-carry": + source, cache = _master_source(args) + if not source: + ap.error("record-audio-carry needs the target master tables: " + "pass --master or --master-url") + from phenomena.record_audio import carry_record_audio + document = carry_record_audio(source, args.manifest, args.source, + args.source_manifest, args.out, + region=args.region, master_cache=cache) + print(json.dumps(document["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-fetch": + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(getattr(args, "unity_version", None)) + from phenomena.record_audio import fetch_record_bundles + receipt = fetch_record_bundles(args.manifest, args.document, args.asset_base_url, + args.out, workers=args.workers) + print(json.dumps(receipt["summary"], ensure_ascii=False)) + return 0 + if args.cmd == "record-audio-merge": + from phenomena.record_audio import merge_record_audio + print(json.dumps(merge_record_audio(args.inputs, args.out)["summary"], + ensure_ascii=False)) + return 0 + if args.cmd == "audio-categories": + from phenomena.audio_categories import write_category_table + document = write_category_table(args.configuration, args.out) + print(json.dumps({"source": document["source"], + "categories": [(row["name"], row["index"], row["volume"]) + for row in document["categories"]]}, + ensure_ascii=False)) + return 0 + if args.cmd == "partvoice-routes": + source, cache = _master_source(args) + if not source: + ap.error("partvoice-routes needs master tables: " + "pass --master or --master-url") + from core.master import Master + from phenomena.partvoice import (build_partvoice_routes, + route_counts, write_partvoice_routes) + roster = json.loads(Path(args.roster).read_text(encoding="utf-8")) + document = build_partvoice_routes(Master(source, cache_dir=cache), + roster) + path = write_partvoice_routes(document, args.out_dir) + print(json.dumps({**route_counts(document), "file": str(path)}, + ensure_ascii=False)) + return 0 + if args.cmd == "material-thumbnails": + from ui.material_thumbnail import export_material_thumbnails + from core.extract import DEFAULT_UNITY_VERSION + result = export_material_thumbnails( + args.bundle, Path(args.master) / "materials.json", args.out_dir, + unity_version=args.unity_version or DEFAULT_UNITY_VERSION) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 if not result["coverage"]["missingMaterialIds"] else 1 + if args.cmd == "honor-images": + from ui.honor_images import export_honor_images + from core.extract import DEFAULT_UNITY_VERSION + result = export_honor_images(args.master, args.bundles, args.out_dir, + unity_version=args.unity_version or DEFAULT_UNITY_VERSION) + print(json.dumps({"counts": result["counts"], "bundles": len(result["bundles"]), + "missingBundles": result["missingBundles"]}, ensure_ascii=False)) + return 0 if not result["missingBundles"] else 1 + if args.cmd == "item-icons": + from ui.item_icon import export_item_icons, package_paths + packages = package_paths(args.decrypted) + masters = {} + if args.root: + for key, name in (("materials", "mysekai-materials.json"), + ("items", "mysekai-items.json"), + ("fixtures", "mysekai-fixtures.json"), + ("tools", "mysekai-tools.json")): + path = Path(args.root) / name + masters[key] = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None + result = export_item_icons(packages, args.out_dir, **masters) + print(json.dumps(result["coverage"], ensure_ascii=False)) + return 0 + if args.cmd == "sitemap-screen-layer": + from sites.screen_layer import extract_screen_layer_jp + result = extract_screen_layer_jp( + args.apk, Path(args.out_dir) / "site" / "sitemap" / "screen_layer", + client_version=args.client_version) + print(json.dumps({"counts": result["counts"], "path": result["path"]}, + ensure_ascii=False)) + return 0 + if args.cmd == "site": + from core.assets.packages import require_paths + from sites.pack import extract_sites + source, cache = _master_source(args) + # Same refusal as the phenomena command: a bare package name under + # --bundle would extract zero objects and report green. + require_paths(args.bundle) + report = extract_sites(args.bundle, args.out_dir, + bundle_root=args.bundle_root, master=source, + master_cache=cache) + print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, + ensure_ascii=False)) + return 0 + if args.cmd == "alone-actions": + from chara.alone_actions import write_alone_actions + print(json.dumps(write_alone_actions(args.bundle, args.out), ensure_ascii=False)) + return 0 + if args.cmd == "facial-tables": + tables = characters.facial_tables(args.bundle) + with open(args.out, "w", encoding="utf-8", newline="\n") as fh: + json.dump(tables, fh, ensure_ascii=False, indent=1, allow_nan=False); fh.write("\n") + print(json.dumps({k: len(v) for k, v in tables.items()})); return 0 + sampled = None + if args.motion_bundle and args.clips: + sampled = characters.sample_clips(args.motion_bundle, [s for s in args.clips.split(",") if s]) + facial = None + if args.facial_tables: + with open(args.facial_tables, encoding="utf-8") as fh: facial = json.load(fh) + cw, ch = args.atlas_cell.lower().split("x") + report = characters.extract_character(args.bundle, args.out_dir, args.name, sampled=sampled, unit_id=args.unit_id, facial=facial, atlas_cell=(int(cw), int(ch)), with_cloth=not args.no_cloth, aux=tuple(args.aux_bundle)) + print(json.dumps(report, ensure_ascii=False)); return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cli/main.py b/src/cli/main.py new file mode 100644 index 0000000..6698481 --- /dev/null +++ b/src/cli/main.py @@ -0,0 +1,18 @@ +"""Grouped public entry point, retaining the existing flat commands.""" +import sys + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] in {"source", "depot"}: + from .pipeline import main as pipeline_main + return pipeline_main(argv) + if argv and argv[0] == "export": + argv = argv[1:] + if argv and argv[0] in {"talks", "communication-actions", "text-catalog", "official-text", "text-runtime"}: + from .pipeline import main as pipeline_main + return pipeline_main(["export", *argv]) + # Existing direct readers stay available through the new command + # group while individual exports move to declared pipeline stages. + from .legacy import main as legacy_main + return legacy_main(argv) diff --git a/src/cli/pipeline.py b/src/cli/pipeline.py new file mode 100644 index 0000000..4699848 --- /dev/null +++ b/src/cli/pipeline.py @@ -0,0 +1,131 @@ +"""Source receipts, reproducible exports and depot inspection.""" +import argparse +import json +from pathlib import Path + +from core.atomic import write_json +from pack.verify import strict_json +from pipeline.depot import Depot +from pipeline.stages import catalog_task, communication_task, official_task, runtime_task, talk_task +from source.snapshot import file_input, read_inputs, snapshot + + +def main(argv): + parser = argparse.ArgumentParser(prog="moly") + groups = parser.add_subparsers(dest="group", required=True) + source = groups.add_parser("source", help="record or verify a versioned source") + sources = source.add_subparsers(dest="command", required=True) + record = sources.add_parser("record") + record.add_argument("--input", action="append", required=True, metavar="ROLE=PATH") + record.add_argument("--region", required=True) + record.add_argument("--version", required=True) + record.add_argument("--out", required=True) + check = sources.add_parser("verify") + check.add_argument("receipt") + export = groups.add_parser("export", help="export products through declared tasks") + exports = export.add_subparsers(dest="command", required=True) + talks = exports.add_parser("talks") + talks.add_argument("--master", required=True) + talks.add_argument("--bundle", required=True) + talks.add_argument("--lib", required=True) + talks.add_argument("--family", choices=("general", "fixture"), default="general") + talks.add_argument("--unity-version", default="2022.3.62f3") + communication = exports.add_parser("communication-actions") + communication.add_argument("--master", required=True) + communication.add_argument("--bundle", required=True) + communication.add_argument("--unity-version", default="2022.3.62f3") + catalog = exports.add_parser("text-catalog") + catalog.add_argument("--corpus", action="append", required=True) + catalog.add_argument("--region", required=True) + catalog.add_argument("--source-revision", required=True) + catalog.add_argument("--previous") + official = exports.add_parser("official-text") + official.add_argument("--source", required=True) + official.add_argument("--donor", required=True) + official.add_argument("--locale", required=True) + runtime = exports.add_parser("text-runtime") + runtime.add_argument("--source", required=True) + runtime.add_argument("--overlay", action="append", default=[]) + for command in (talks, communication, catalog, official, runtime): + command.add_argument("--depot", required=True) + command.add_argument("--out", required=True, help="layout directory, outside the depot") + command.add_argument("--plan", action="store_true", help="inspect reuse without extraction or layout") + command.add_argument("--emit-task", help="save the portable task description") + command.add_argument("--force", action="store_true", help="recompute and compare the existing result") + depot = groups.add_parser("depot", help="inspect, run, verify and lay out tasks") + depots = depot.add_subparsers(dest="command", required=True) + for name in ("plan", "run", "verify", "layout", "gc"): + command = depots.add_parser(name) + command.add_argument("--depot", required=True) + if name in ("plan", "run"): + command.add_argument("--task", required=True) + if name == "plan": + command.add_argument("--previous", help="previous task for invalidation reasons") + if name == "run": + command.add_argument("--force", action="store_true") + if name == "layout": + command.add_argument("--key", required=True) + command.add_argument("--out", required=True) + if name == "gc": + command.add_argument("--apply", action="store_true") + args = parser.parse_args(argv) + try: + if args.group == "source": + if args.command == "record": + inputs = [] + for item in args.input: + role, separator, path = item.partition("=") + if not separator or not role or not path: + raise ValueError("--input expects ROLE=PATH") + inputs.append(file_input(role, path)) + result = snapshot(inputs, region=args.region, version=args.version) + write_json(args.out, result) + report = {"revision": result["revision"], "inputs": len(inputs)} + else: + result = strict_json(Path(args.receipt).read_bytes()) + rebuilt = snapshot(result["inputs"], region=result["region"], version=result["version"]) + if result != rebuilt: + raise ValueError("source receipt identity differs from its inputs") + read_inputs(result["inputs"]) + report = {"revision": result["revision"], "verified": True} + elif args.group == "export": + if args.command == "talks": + task = talk_task(args.master, args.bundle, args.lib, family=args.family, + unity_version=args.unity_version) + elif args.command == "communication-actions": + task = communication_task(args.master, args.bundle, unity_version=args.unity_version) + elif args.command == "text-catalog": + task = catalog_task(args.corpus, region=args.region, source_revision=args.source_revision, + previous=args.previous) + elif args.command == "official-text": + task = official_task(args.source, args.donor, locale=args.locale) + else: + task = runtime_task(args.source, args.overlay) + depot = Depot(args.depot) + if args.emit_task: + write_json(args.emit_task, task) + if args.plan: + report = depot.plan(task) + else: + run = depot.run(task, force=args.force) + report = {"status": run["status"], **depot.materialize(task["key"], args.out)} + else: + depot = Depot(args.depot) + if args.command in {"plan", "run"}: + task = strict_json(Path(args.task).read_bytes()) + if args.command == "plan": + previous = strict_json(Path(args.previous).read_bytes()) if args.previous else None + report = depot.plan(task, previous=previous) + else: + report = depot.run(task, force=args.force) + elif args.command == "layout": + report = depot.materialize(args.key, args.out) + elif args.command == "gc": + from pack.gc import collect_garbage + report = collect_garbage(args.depot, apply=args.apply) + else: + report = depot.verify() + print(json.dumps(report, ensure_ascii=False)) + return 0 + except (OSError, ValueError, KeyError, TypeError) as exc: + parser.exit(2, f"moly: {exc}\n") diff --git a/src/core/assets/router.py b/src/core/assets/router.py index cc22c25..d84ade9 100644 --- a/src/core/assets/router.py +++ b/src/core/assets/router.py @@ -187,6 +187,8 @@ def route(name): return Route("facial", "facial-tables") if name == "mysekai__character_alone_action": return Route("performance", "alone-actions") + if name == "mysekai__some_character_communication": + return Route("communication", "communication-actions") if "__effect__emoticon__" in name: return Route("emoticon", "emoticons") if name == "mysekai__talk__scenario__talk": diff --git a/src/core/cli.py b/src/core/cli.py index 7e5378c..1eea1aa 100644 --- a/src/core/cli.py +++ b/src/core/cli.py @@ -1,1688 +1,7 @@ -"""Single command-line entry point for bundle extraction.""" -import argparse -import os -import warnings -import json +"""Compatibility entry point; command implementation lives in cli.""" import sys -from pathlib import Path - -# Extraction output root; MOLY_EXTRACT_OUT lets a workspace keep it outside the checkout. -DEFAULT_DATA_ROOT = os.environ.get("MOLY_EXTRACT_OUT") or "local-data" - - -PLAYER_DATA_HELP = ("the player data: the container file (assets/bin/Data/data.unity3d), " - "or, for a build without one, each APK whose assets/bin/Data members " - "are its loose player files; repeat for the base APK and the asset pack") - - -def _configure_unity(fallback_version): - import UnityPy.config - if fallback_version: - UnityPy.config.FALLBACK_UNITY_VERSION = fallback_version - - -def _master_source(args): - """Directory or base URL for the caller's master tables, plus a cache dir. - - ``--master`` names a local directory; ``--master-url`` fetches each table by - appending its name to a base URL (with no value, the public default base). - Neither is implied: without one of them nothing is read and nothing is - fetched. - """ - from .master import DEFAULT_MASTER_URL - url = getattr(args, "master_url", None) - if url is not None: - return url or DEFAULT_MASTER_URL, getattr(args, "master_cache", None) - return getattr(args, "master", None), None - - -def _write_programs(directory, entries, platform=None): - """Write each program body to its own file; return how many were written. - - A body is named by shader, platform and record index rather than by content - hash, so a caller reading a file knows which variant produced it; identical - variants therefore appear more than once, which is the honest shape -- the - census reports the distinct count separately. - """ - directory = Path(directory) - directory.mkdir(parents=True, exist_ok=True) - written = 0 - for entry in entries: - stem = "".join(c if c.isalnum() or c in "-_" else "_" for c in entry["shader"]) - for block in entry["platforms"]: - if platform is not None and block["platform"] != platform: - continue - for row in block["programs"]: - suffix = "glsl" if row["isText"] else "bin" - name = f"{stem}__platform{block['platform']}__record{row['record']}.{suffix}" - (directory / name).write_bytes(row.get("code") or b"") - written += 1 - return written - - -def _extract_failed(report): - """A failed derived artifact is a failed extraction, not a skipped option. - - So is a failed player-data artifact: its input was supplied and it could - not be read, which is a failure of the run, while a player-data artifact - whose input nobody supplied stays ``skipped`` and is named as an input gap. - """ - return bool(report["summary"]["failed"]) or any( - entry.get("status") == "failed" - for entry in report.get("derived", []) + report.get("playerData", [])) - - -def _print_extract_summary(report, out_dir): - s = report["summary"] - print(f"extracted {s['succeeded']}/{s['requested']} bundles -> {out_dir} " - f"(failed {s['failed']}, unsupported {s['unsupported']})") - for entry in report["bundles"]: - if entry["status"] == "failed": - print(f" failed: {entry['bundle']}: {entry['error']}") - for entry in report.get("derived", []) + report.get("playerData", []): - if entry.get("status") == "failed": - print(f" failed artifact: {entry['artifact']}: {entry.get('error', '')}") - programs = entry.get("sourcePrograms") - if programs: - print(f" particle source programs: {programs['status']} " - f"({programs['converted']}/{programs['gles3Variants']} GLES3 variants converted, " - f"unavailable {programs['unavailable']})") - print(f" {programs['note']}") - # An artifact whose input nobody supplied is not a failure of the run, but - # it is not silent either: each one is named with its reason (a failed one - # is already named above). - for gap in (report.get("inputGaps") or {}).get("unaccepted", []): - if gap.get("why") != "failed": - print(f" input gap: {gap['artifact']}: {gap.get('detail', '')}") - print(f"report: {report['report']}") - if str(out_dir) == "local-data": - print("view: python -m http.server 8000 (from the repository root), then open\n" - " http://localhost:8000/examples/viewer/index.html?base=../../local-data") - else: - print("view: serve the repository root over HTTP and open " - "examples/viewer/index.html?base=") - - -def main(argv=None): - from chara.rotation_encoding import DEFAULT_ROTATION_ENCODING, ROTATION_ENCODINGS - # UnityPy warns about asset shapes it does not model. Silencing that is - # this command's choice to make, so it is made here, once, for this - # process. It used to sit at module scope in four extractors, where - # importing one rewrote the warning filters of any program that used this - # package as a library -- including filters that program had set itself. - warnings.filterwarnings("ignore") - ap = argparse.ArgumentParser(prog="moly", description="Extract Unity humanoid assets") - ap.add_argument("--unity-version", default=None) - sub = ap.add_subparsers(dest="cmd", required=True) - p = sub.add_parser("pull", help="download, decrypt, and extract character asset packs") - p.add_argument("--manifest", required=True) - p.add_argument("--out", help="download/decrypt workspace (default: moly-pull-output)") - p.add_argument("--extract-out", default=None, - help="where extraction artifacts go (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") - p.add_argument("--json", action="store_true", - help="print the full JSON report to stdout instead of a summary") - p.add_argument("--asset-base-url", required=True) - p.add_argument("--workers", type=int, default=8) - p.add_argument("--retries", type=int, default=4) - p.add_argument("--master", help="directory of caller-supplied master tables") - p.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - p.add_argument("--master-cache", help="where fetched tables are cached") - p.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - p.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") - x = sub.add_parser("extract", help="extract bundles listed in a manifest") - x.add_argument("--manifest", - help="optional bundle list; omitted = select every supported " - "asset-pack bundle found under --bundles") - x.add_argument("--bundles", required=True) - x.add_argument("--out", default=DEFAULT_DATA_ROOT, - help="output directory (default: $MOLY_EXTRACT_OUT, else local-data, which stays out of version control)") - x.add_argument("--json", action="store_true", - help="print the full JSON report to stdout instead of a summary") - x.add_argument("--master", help="directory of caller-supplied master tables") - x.add_argument("--player-data", action="append", - help="the APK player data: the container file " - "(assets/bin/Data/data.unity3d), or, for a build " - "without one, each APK whose assets/bin/Data members " - "are its loose player files (repeat: the base APK and " - "the asset pack). It supplies the built-in UI and " - "screen-layer assets that the download path does not " - "carry (container builds only) and the Resources " - "objects: the live transitioner, the effect type " - "table, the graphics configuration, the layout grid") - x.add_argument("--player-types", metavar="DIR", - help="managed assembly stubs of the same player build (a DummyDll " - "directory); the Resources objects read from --player-data " - "(the live transitioner, the effect type table, the drop-item " - "shading) need them, and without them those artifacts are " - "skipped with the reason") - x.add_argument("--player-declarations", metavar="FILE", - help="the same build's declaration dump; names the effect types in " - "effect-resources/ (without it the names are null)") - x.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - x.add_argument("--master-cache", help="where fetched tables are cached") - x.add_argument("--fixture-meshes", action="store_true", - help="also write furniture geometry (fixture-models/), one " - "glTF binary per package; without it that pass is " - "skipped and the passes that need the geometry report " - "it as a missing dependency") - x.add_argument("--fixture-particles", action="store_true", - help="also write furniture particle emitters " - "(fixture-particles/), one document per package " - "across the fixture-interface, cutscene-timeline and " - "fixture-timeline families; without it that pass is " - "skipped. Playable output requires --shader-compiler; " - "without it only source data is saved, not playable assets") - x.add_argument("--shader-compiler", metavar="PATH", - help="pinned offline source shader compiler required for playable " - "furniture particles; omitting it saves source-only data") - x.add_argument("--builtin-resources", action="append", default=[], - metavar="PATH", - help="the engine's own built-in resource container, or a " - "directory holding it; some particle-system renderers " - "and some phenomena emitters draw a copy of a " - "built-in primitive mesh and no package ships those. " - "Repeatable. Without it those mesh slots stay listed " - "as unresolved in fixture-particles/ and in the " - "phenomena index, exactly as before") - x.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - x.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed copy of each decoded sound") - x.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") - c = sub.add_parser("characters", help="extract one character bundle") - c.add_argument("--bundle", required=True); c.add_argument("--out-dir", required=True); c.add_argument("--name", required=True) - c.add_argument("--aux-bundle", action="append", default=[]); c.add_argument("--motion-bundle"); c.add_argument("--clips"); c.add_argument("--unit-id", type=int); c.add_argument("--facial-tables"); c.add_argument("--atlas-cell", default="512x256"); c.add_argument("--no-cloth", action="store_true") - m = sub.add_parser("motion-library", help="export the shared humanoid motion library") - m.add_argument("--reference-bundle", required=True); m.add_argument("--motion-bundle", required=True); m.add_argument("--aux-bundle", action="append", default=[]); m.add_argument("--clips"); m.add_argument("--out-dir", required=True); m.add_argument("--name", default="motion-library") - m.add_argument("--controller-bundle", action="append", default=None, - help="character package whose AnimatorController selects the clips " - "(by serialized identity); repeat per package. Without it the " - "first clip of each name is kept and duplicates are listed") - m.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="rotation key storage (default: snorm16); float32 disables lossy encoding") - t = sub.add_parser("facial-tables", help="dump facial tables") - t.add_argument("--bundle", required=True); t.add_argument("--out", required=True) - n = sub.add_parser("alone-actions", help="dump per-character performance data (motion x facial x timing)") - n.add_argument("--bundle", required=True); n.add_argument("--out", required=True) - g = sub.add_parser("registry", help="build the character registry from master tables") - g.add_argument("--master", help="directory of caller-supplied master tables") - g.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - g.add_argument("--master-cache", help="where fetched tables are cached") - g.add_argument("--manifest", required=True, help="manifest naming the character bundles") - g.add_argument("--motion-index", help="shared motion library index, to check motion names") - g.add_argument("--out", required=True) - - k = sub.add_parser("talks", help="extract single-character direct-talk corpus") - k.add_argument("--master", help="directory of caller-supplied master tables") - k.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - k.add_argument("--master-cache", help="where fetched tables are cached") - k.add_argument("--bundle", required=True, help="talk scenario bundle") - k.add_argument("--lib", help="decrypted lib package carrying the constant " - "tables the scripts name; without it every " - "Table.key token stays a source token") - k.add_argument("--out", required=True) - - tw = sub.add_parser("tweets", help="extract tweet texts and per-character after-edit pools") - tw.add_argument("--master", help="directory of caller-supplied master tables") - tw.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - tw.add_argument("--master-cache", help="where fetched tables are cached") - tw.add_argument("--out", required=True) - - tt = sub.add_parser("tweet-tables", - help="extract the tweet selection tables (greetings, " - "greeting conditions, site entries, talk pre-actions)") - tt.add_argument("--master", help="directory of caller-supplied master tables") - tt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - tt.add_argument("--master-cache", help="where fetched tables are cached") - tt.add_argument("--out", required=True) - - sg = sub.add_parser("site-groups", - help="extract the mysekai site-group table " - "(siteGroupId -> the site ids it holds)") - sg.add_argument("--master", help="directory of caller-supplied master tables") - sg.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - sg.add_argument("--master-cache", help="where fetched tables are cached") - sg.add_argument("--out", required=True) - - bp = sub.add_parser("birthday-parties", - help="extract the birthday-party campaign table the " - "site-map festival gate reads (startAt/closedAt windows)") - bp.add_argument("--master", help="directory of caller-supplied master tables") - bp.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bp.add_argument("--master-cache", help="where fetched tables are cached") - bp.add_argument("--out", required=True) - - bd = sub.add_parser("birthday-party-delivery", - help="extract the birthday-party delivery tables the " - "delivery site reads (rewards, point bonuses, total rewards)") - bd.add_argument("--master", help="directory of caller-supplied master tables") - bd.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bd.add_argument("--master-cache", help="where fetched tables are cached") - bd.add_argument("--out", required=True) - - gv = sub.add_parser("gate-visit-policy", - help="extract the gate visit tables as policy data for the " - "server mock (not a client character input)") - gv.add_argument("--master", help="directory of caller-supplied master tables") - gv.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - gv.add_argument("--master-cache", help="where fetched tables are cached") - gv.add_argument("--out", required=True) - - tm = sub.add_parser("time-manager", - help="extract the project TimeManager settings (fixed step, " - "maximum allowed timestep, time scale, particle step) " - "from an APK or a globalgamemanagers file") - tm.add_argument("--source", required=True, - help="an APK, or a player-data globalgamemanagers file") - tm.add_argument("--out", required=True) - - cc = sub.add_parser("client-config", - help="extract the ClientConfig deliverable panel " - "(the four typed dictionaries, keyed by id)") - cc.add_argument("--master", help="directory of caller-supplied master tables") - cc.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - cc.add_argument("--master-cache", help="where fetched tables are cached") - cc.add_argument("--out", required=True) - - from core.master_tables import REGISTERED_TABLES - mt = sub.add_parser("master-tables", - help="extract selected key-addressed mysekai master " - "tables, defaulting to blueprints, items, music " - "records and wordings") - mt.add_argument("--master", help="directory of caller-supplied master tables") - mt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - mt.add_argument("--master-cache", help="where fetched tables are cached") - mt.add_argument("--table", dest="tables", action="append", - choices=tuple(REGISTERED_TABLES), - help="registered table to export; repeat to select more " - "(omitting this exports only the original four tables)") - mt.add_argument("--out", required=True, - help="directory the selected documents are written into") - - e = sub.add_parser("emoticons", help="extract overhead-item effect packages") - e.add_argument("--bundle", action="append", required=True); e.add_argument("--out-dir", required=True) - v = sub.add_parser("avatar-parts", help="extract player-appearance packages (skin/decoration/penlight)") - v.add_argument("--bundle", action="append", required=True); v.add_argument("--out-dir", required=True) - ht = sub.add_parser("harvest-tools", help="extract original harvest-tool prefabs and their material data") - ht.add_argument("--bundle", action="append", required=True) - ht.add_argument("--bundle-root", help="directory containing declared shader dependencies") - ht.add_argument("--out-dir", required=True, help="tool directory, normally /avatar/tools") - ptm = sub.add_parser("player-tool-models", - help="extract the avatar's hand-item prefabs and their material data " - "(mysekai/player_tool_model/*)") - ptm.add_argument("--bundle", action="append", required=True) - ptm.add_argument("--bundle-root", help="directory containing declared shader dependencies") - ptm.add_argument("--out-dir", required=True, - help="model directory, normally /avatar/player-tool-models") - er = sub.add_parser("effect-resources", - help="export the effect manager's type table " - "(Resources Mysekai/Effects/EffectResourceData) from player data") - er.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - er.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - er.add_argument("--declarations", - help="the same build's declaration dump, for effect type names") - er.add_argument("--bundle-root", - help="bundle directory to check each row's bundle and prefab against") - er.add_argument("--out", required=True, help="output JSON path") - dis = sub.add_parser("drop-item-shade", - help="export the drop-item shading settings of the graphics " - "configuration (Resources Graphics/Data/MysekaiGraphicsConfig)") - dis.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - dis.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - dis.add_argument("--out", required=True, help="output JSON path") - sts = sub.add_parser("stencil-shadow", - help="export the character stencil-shadow colour and the material " - "rendering settings of the graphics configuration") - sts.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - sts.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - sts.add_argument("--out", required=True, help="output JSON path") - lgr = sub.add_parser("layout-grid", - help="export the layout editor's placement grid: its settings, " - "materials, shader programs and colour profiles " - "(Resources Data/MysekaiGridSettings and " - "Data/GridColorProfileTable)") - lgr.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - lgr.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - lgr.add_argument("--out-dir", required=True, - help="output directory, normally /layout-grid") - rpf = sub.add_parser("resource-prefab", - help="extract one prefab the player data ships in Resources, by load name") - rpf.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - rpf.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - rpf.add_argument("--load-name", required=True, help="the Resources.Load path") - rpf.add_argument("--out-dir", required=True, - help="directory receiving /.{json,glb} and textures") - rpf.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", - help="the engine's built-in resource file (unity default resources, " - "unity_builtin_extra) or a directory holding them; a container " - "build keeps them beside data.unity3d; repeat for each") - rpr = sub.add_parser("resource-programs", - help="write the compiled programs of every shader a Resources " - "prefab's materials use (the program library layout)") - rpr.add_argument("--player-data", required=True, action="append", - help=PLAYER_DATA_HELP) - rpr.add_argument("--types", required=True, - help="managed assembly stubs of the same build (DummyDll directory)") - rpr.add_argument("--load-name", required=True, help="the Resources.Load path") - rpr.add_argument("--out-dir", required=True, - help="directory receiving /shaders/ (the resource-prefab out-dir)") - rpr.add_argument("--shader-compiler", - help="offline shader compiler executable; without it the programs are " - "written as source only (no conversions)") - fp = sub.add_parser("fixture-particles", - help="write particle source-schema documents (the fixture-particles " - "layout) for explicitly named bundles") - fp.add_argument("--bundle", action="append", required=True) - fp.add_argument("--bundle-root", help="directory the bundles' dependencies are read from") - fp.add_argument("--domain", action="append", - help="routed domain to extract; repeat; default the three furniture domains") - fp.add_argument("--shader-compiler", metavar="PATH", - help="pinned offline shader compiler; without it programs stay source-only") - fp.add_argument("--builtin-resources", action="append", default=[], metavar="PATH", - help="the engine's built-in resource container (or its directory), " - "for renderers that point at a built-in mesh") - fp.add_argument("--out-dir", required=True) - pp = sub.add_parser("prefab-package", - help="extract one named prefab of a bundle with the site package reader") - pp.add_argument("--bundle", required=True, help="the bundle file") - pp.add_argument("--prefab", required=True, - help="the prefab's container basename, e.g. fieldcamera.prefab") - pp.add_argument("--bundle-root", help="directory the bundle's dependencies are read from") - pp.add_argument("--out-dir", required=True, - help="directory receiving /.{json,glb} and textures") - sae = sub.add_parser("site-action-effects", - help="extract the site-move and harvest-action effect prefabs " - "(mysekai/effect/site/harvest/action/*)") - sae.add_argument("--bundle", action="append", required=True) - sae.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - sae.add_argument("--out-dir", required=True, - help="effect directory, normally /site-action-effects") - she = sub.add_parser("site-home-effects", - help="extract the home-site effect prefabs, the craft effect among " - "them (mysekai/effect/site/home/*)") - she.add_argument("--bundle", action="append", required=True) - she.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - she.add_argument("--out-dir", required=True, - help="effect directory, normally /site-home-effects") - fpe = sub.add_parser("fixture-put-effects", - help="extract the furniture placement effect prefab " - "(mysekai/effect/fixture/fx_fixture_put_01)") - fpe.add_argument("--bundle", action="append", required=True) - fpe.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - fpe.add_argument("--out-dir", required=True, - help="effect directory, normally /fixture-put-effects") - ssw = sub.add_parser("site-sound-wave", - help="resolve MysekaiSiteView._soundWaveMeshFilter (the footstep " - "surface) of site scene packages: node, mesh and vertex colours") - ssw.add_argument("--bundle", action="append", required=True, - help="a mysekai__site__field__ package; repeat") - ssw.add_argument("--bundle-root", help="directory the scene packages' dependencies are read from") - ssw.add_argument("--out", required=True, help="output JSON path") - sce = sub.add_parser("site-common-action-effects", - help="extract the player's foot effect prefabs " - "(mysekai/effect/site/common/action/*)") - sce.add_argument("--bundle", action="append", required=True) - sce.add_argument("--bundle-root", - help="directory containing declared effect, texture and shader dependencies") - sce.add_argument("--out-dir", required=True, - help="effect directory, normally /site-common-action-effects") - sho = sub.add_parser("site-harvest-object-effects", - help="extract the harvest-object hit, delete and stay effect prefabs " - "(mysekai/effect/site/harvest/object//*)") - sho.add_argument("--bundle", action="append", required=True) - sho.add_argument("--bundle-root", - help="directory containing declared effect, mesh, texture and shader dependencies") - sho.add_argument("--out-dir", required=True, - help="effect directory, normally /site-harvest-object-effects") - pa = sub.add_parser("player-avatar", - help="export the player avatar composite (audience model + skeleton + motion clips) as one glb") - pa.add_argument("--bundle", action="append", required=True, - help="the avatar model bundle first, then every motion bundle whose clips " - "are to be resolved against it (the order export_player_avatar takes); " - "repeat --bundle per package") - pa.add_argument("--out-dir", required=True) - pa.add_argument("--name", default="mysekai__player_avatar", - help="output basename (glb + .index.json + .motion-manifest.json)") - w = sub.add_parser("phenomena", help="extract weather (phenomena) environment packages") - w.add_argument("--bundle", action="append", required=True, - help="an environment package, or the shared phenomena thumbnail package") - w.add_argument("--bundle-root", - help="directory the packages a bundle depends on are read from") - w.add_argument("--builtin-resources", action="append", default=[], - metavar="PATH", - help="the engine's own built-in resource container, or a " - "directory holding it; some emitters draw copies of a " - "built-in primitive and no package ships those. Repeatable. " - "Without it those meshes stay listed under `unsupported`, " - "exactly as before") - w.add_argument("--shader-compiler", metavar="PATH", - help="moly-shader-bridge executable; without it source programs " - "are preserved and their conversion is explicitly unavailable") - w.add_argument("--site-bundle", action="append", default=None, metavar="PATH", - help="a site scene package whose site prefab colliders go into " - "collision.json; repeatable. Without it every scene package " - "under --bundle-root is used") - w.add_argument("--out-dir", required=True) - w.add_argument("--master", help="directory of caller-supplied master tables") - w.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - w.add_argument("--master-cache", help="where fetched tables are cached") - w.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it; " - "without it PATH is searched and the audio " - "entry says what is missing") - w.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - a = sub.add_parser("mysekai-audio", - help="extract the rest of the sound corpus: talk voices " - "named by talks.json, plus the remaining se, bgm " - "and part-voice packages") - a.add_argument("--talks", required=True, - help="a talk corpus in either shape this repo's extractors " - "write (talks grouped under units, or a flat talks " - "list); its voice cues are the talk-voice denominator, " - "and only those cues are decoded") - a.add_argument("--manifest", required=True, - help="asset bundle manifest giving the family denominators") - a.add_argument("--bundle-root", required=True, - help="directory holding the decrypted sound packages") - a.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "the products and whose loop.json is merged into") - a.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - a.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - ca = sub.add_parser("cutscene-audio", - help="decode the cues a cut-scene's own timeline names: its " - "voice clips from the cut-scene's birthday voice bank, " - "its music and effect clips from the packages they name") - ca.add_argument("--clips", required=True, action="append", - help="a cut-scene clip document (cutscene-timeline/clips/" - ".json); repeat for each cut-scene") - ca.add_argument("--cut-scenes", required=True, - help="the cut-scene master table (mysekaiCutScenes.json), " - "which routes voice clips to their voice bank") - ca.add_argument("--manifest", required=True, - help="asset bundle manifest the named packages are checked against") - ca.add_argument("--bundle-root", required=True, - help="directory holding the decrypted sound packages") - ca.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "the products and whose loop.json and corpus.json are merged into") - ca.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - ca.add_argument("--ffmpeg", help="path to ffmpeg, used only to write a compressed " - "copy of each decoded sound") - sq = sub.add_parser("audio-sequences", - help="attach the cues' sequence structure (type, repeat, " - "per-track delay, track -> stream) to an existing " - "loop sidecar, reading the archives again from the " - "decrypted tree; writes a copy, never in place") - sq.add_argument("--loop", required=True, help="the loop sidecar to read") - sq.add_argument("--bundle-root", required=True, action="append", - help="directory holding the decrypted sound packages; repeat " - "to give further trees, tried in order per entry") - sq.add_argument("--out", required=True, - help="where the annotated copy is written (must not be --loop)") - ra = sub.add_parser("record-audio", - help="read the waveform facts (length, loop region or none, " - "cue commands) of every package a music record can " - "play; the audio itself is not written") - ra.add_argument("--master", help="directory of caller-supplied master tables") - ra.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - ra.add_argument("--master-cache", help="where fetched tables are cached") - ra.add_argument("--bundle-root", required=True, action="append", - help="directory holding decrypted bundles; repeat to give further trees") - ra.add_argument("--bundle-version", required=True, - help="the asset version the bundles were downloaded at (written on every row)") - ra.add_argument("--only", choices=("music", "music_sound_track"), - help="read only the packages of this record track type") - ra.add_argument("--vgmstream", help="path to the external audio decoder " - "(vgmstream-cli), or the directory holding it") - ra.add_argument("--out", required=True, help="the document to write") - rs = sub.add_parser("record-audio-storage", - help="add each row's storage mirror audio file: sha256 and size of " - "the downloaded bytes and the listing's version") - rs.add_argument("input", help="a record-audio document") - rs.add_argument("--server", required=True, choices=("jp", "cn")) - rs.add_argument("--workers", type=int, default=4) - rs.add_argument("--out", required=True) - rm = sub.add_parser("record-audio-merge", - help="combine record-audio documents of separate runs") - rm.add_argument("inputs", nargs="+") - rm.add_argument("--out", required=True) - rc = sub.add_parser("record-audio-carry", - help="carry a read record-audio document's facts to another " - "region's packages whose bundles have the same crc and size") - rc.add_argument("--master", help="directory of the target region's master tables") - rc.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - rc.add_argument("--master-cache", help="where fetched tables are cached") - rc.add_argument("--manifest", required=True, - help="the target region's package list (AssetBundleInfoNew)") - rc.add_argument("--source", required=True, - help="a record-audio document with storage attached") - rc.add_argument("--source-manifest", required=True, - help="the package list of the bundles the source document was read from") - rc.add_argument("--region", required=True, choices=("jp", "cn"), - help="the target region, written on the document") - rc.add_argument("--out", required=True) - rf = sub.add_parser("record-audio-fetch", - help="download and check (size, uncompressed crc) the bundles of " - "the packages a record-audio document lists as absent") - rf.add_argument("document", help="a record-audio document (its absent list is fetched)") - rf.add_argument("--manifest", required=True, - help="the package list (AssetBundleInfoNew) the bundles are fetched by") - rf.add_argument("--asset-base-url", required=True, - help="the prefix before each entry's downloadPath") - rf.add_argument("--workers", type=int, default=4) - rf.add_argument("--out", required=True, help="workspace; bundles land in /decrypted") - ac = sub.add_parser("audio-categories", - help="write the sound configuration's category table (name, " - "index, volume) the runtime scales cues by") - ac.add_argument("--configuration", required=True, - help="the sound configuration (.acf), or the application " - "package (zip) that holds assets/*.acf") - ac.add_argument("--out", required=True, - help="where categories.json is written (the runtime reads " - "phenomena/audio/categories.json)") - r = sub.add_parser("partvoice-routes", - help="build the speaker-to-partvoice-package routing " - "table from master tables and the roster") - r.add_argument("--master", help="directory of caller-supplied master tables") - r.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - r.add_argument("--master-cache", help="where fetched tables are cached") - r.add_argument("--roster", required=True, - help="characters.json, the registry artifact; its " - "characters are the participants' universe and order") - r.add_argument("--out-dir", required=True, - help="the phenomena output directory whose audio/ receives " - "partvoice.json") - u = sub.add_parser("ui-action-icons", - help="export the standalone action-button icon bundles " - "as one PNG per bundle plus a manifest") - u.add_argument("--bundle", action="append", required=True, - help="an action-icon bundle; repeat for as many as wanted") - u.add_argument("--out-dir", required=True) - portraits = sub.add_parser("browser-assets", - help="export source character portraits for the browser catalog") - portraits.add_argument("--player-data", required=True, - help="the APK player data containing CharacterSDIconAtlas") - portraits.add_argument("--asset-root", required=True, - help="runtime resource root containing characters.json") - portraits.add_argument("--source-region", required=True, choices=("cn", "jp")) - portraits.add_argument("--source-game-version", required=True, - help="game version of the supplied UI player data") - presentation = sub.add_parser("fixture-presentation", - help="resolve model, wallpaper, floor and custom furniture source routes") - presentation.add_argument("--asset-root", required=True) - presentation.add_argument("--master", required=True) - presentation.add_argument("--bundles", required=True) - validate = sub.add_parser("validate-runtime", help="verify referenced resources and exact audio cue coverage") - validate.add_argument("--asset-root", required=True) - validate.add_argument("--verify-actor-hashes", action="store_true") - fixture_animations = sub.add_parser("fixture-animations", - help="refresh source-bound fixture clips and their authored time domain") - fixture_animations.add_argument("--asset-root", required=True) - fixture_animations.add_argument("--bundles", required=True) - fixture_animations.add_argument("--package", action="append") - fixture_animations.add_argument("--plan", action="store_true") - capabilities = sub.add_parser("timeline-capabilities", help="index source payload requirements per fixture prefab") - capabilities.add_argument("--asset-root", required=True) - lay = sub.add_parser("ui-layout", - help="extract the RectTransform hierarchy of the UI " - "prefabs (screen layers, dialogs) the player data " - "builds, plus a census of the UI package families") - lay.add_argument("--region", choices=("cn", "jp"), default="cn", - help="cn: one player data file whose resources.assets holds " - "every prefab; jp: the base APK and the asset pack, " - "whose Resources index points at one serialized file " - "per prefab") - lay.add_argument("--player-data", default=None, - help="cn: the APK player data file; every screen and dialog " - "prefab lives in its resources.assets (Resources.Load " - "fetches them, so no bundle name routes to them)") - lay.add_argument("--apk", action="append", default=None, - help="jp: an APK whose assets/bin/Data members are player " - "files; give the base APK and the asset pack") - lay.add_argument("--client-version", default=None, - help="jp: the client version the manifest and every document " - "name (ui/jp_decoders.py names the supported ones; each " - "object's chain comes from its serialized type hash); cn: " - "the client version the palette and tween defaults name") - lay.add_argument("--master", default=None, - help="jp: directory of master tables holding wordings.json") - lay.add_argument("--prefab", action="append", default=None, - help="a prefab name to extract (repeatable); a targeted " - "run writes those documents and omits the broad census") - lay.add_argument("--out-dir", required=True, - help="the output directory (ui-layout/) that receives one " - "document per prefab plus census.json") - lay.add_argument("--bundles", default=None, - help="optional decrypted-package directory; named so the " - "census also reports what the per-screen download " - "packages contain (textures and effects, not layout)") - lay.add_argument("--font-bundles", default=None, - help="jp: decrypted-package directory holding font/common " - "(jp download --root font/common); defaults to --bundles") - lay.add_argument("--bundle-manifest", default=None, - help="optional AssetBundleInfoNew.json; supplies the " - "isBuiltin split and declared sizes for the census") - words = sub.add_parser("ui-wordings", - help="write the runtime wording dictionary of a container build: " - "the built-in wording CSV of its language, then the master " - "wordings over it (the document a shared root reads as its " - "wordings.json)") - words.add_argument("--player-data", required=True, - help="the APK player data file whose Resources index holds the wording CSV") - words.add_argument("--master", required=True, - help="directory of master tables holding wordings.json") - words.add_argument("--client-version", required=True, - help="the client version the document's source block names") - words.add_argument("--out", required=True, help="the document path") - reactions = sub.add_parser("fixture-reactions", - help="export the native furniture reaction-character groups") - reactions.add_argument("--bundle", required=True, - help="decrypted mysekai__system__fixture_reaction_data bundle") - reactions.add_argument("--out", required=True, - help="output fixture-reactions.json path") - bem = sub.add_parser("birthday-end-motions", - help="export the waving motion the birthday party " - "plays as it closes (row per character unit)") - bem.add_argument("--bundle", required=True, - help="decrypted mysekai__birthday__birthday_end_character_motion_data_list bundle") - bem.add_argument("--out", required=True, - help="output birthday-end-character-motions.json path") - bpt = sub.add_parser("birthday-party-talks", - help="attach the birthday-party scripts one master " - "table names to an existing talks.json") - bpt.add_argument("--master", help="directory of caller-supplied master tables") - bpt.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - bpt.add_argument("--master-cache", help="where fetched tables are cached") - bpt.add_argument("--bundle", required=True, help="talk scenario bundle") - bpt.add_argument("--lib", help="decrypted lib package carrying the constant " - "tables the scripts name") - bpt.add_argument("--talks", required=True, - help="an existing talks.json whose birthdayParty block is " - "computed from this master partition") - bpt.add_argument("--out", required=True) - f = sub.add_parser("fixture-master-slice", - help="slice mysekaiFixtures down to the columns the " - "runtime reads (button gating and footprint)") - f.add_argument("--master", help="directory of caller-supplied master tables") - f.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - f.add_argument("--master-cache", help="where fetched tables are cached") - f.add_argument("--out", required=True) - q = sub.add_parser("site", help="extract the site (place) asset packages") - q.add_argument("--bundle", action="append", required=True, - help="a package under the site path; repeat for as many as wanted") - q.add_argument("--bundle-root", - help="directory the packages a bundle depends on are read from") - q.add_argument("--out-dir", required=True) - q.add_argument("--master", help="directory of caller-supplied master tables") - q.add_argument("--master-url", nargs="?", const="", default=None, - help="base URL to append
.json to; no value uses the public default base") - q.add_argument("--master-cache", help="where fetched tables are cached") - ua = sub.add_parser("ui-audio", help="extract embedded UI cue sheets with the shared audio decoder") - ua.add_argument("--apk", required=True) - ua.add_argument("--out", required=True) - ua.add_argument("--vgmstream") - ua.add_argument("--ffmpeg") - sls = sub.add_parser("sitemap-screen-layer", - help="write site/sitemap/screen_layer/screen_layer.json from APK " - "members (player data without resources.assets)") - sls.add_argument("--apk", action="append", required=True, - help="an APK whose assets/bin/Data members are player files; " - "give the base APK and the asset pack") - sls.add_argument("--client-version", required=True, - help="the client version the field chains are for") - sls.add_argument("--out-dir", required=True, - help="runtime root; the document lands at " - "site/sitemap/screen_layer/screen_layer.json under it") - mth = sub.add_parser("material-thumbnails", - help="write material-thumbnails.json and textures/material{id}.png " - "from the thumbnail/material package, one per materials row") - mth.add_argument("--bundle", required=True, help="the decrypted thumbnail/material package") - mth.add_argument("--master", required=True, - help="directory of master tables holding materials.json") - mth.add_argument("--out-dir", required=True, - help="output directory (material-thumbnails/ of a runtime root)") - hon = sub.add_parser("honor-images", - help="write honor/index.json and the textures of the honor bundles " - "the birthday-party delivery rewards' honors draw") - hon.add_argument("--master", required=True, - help="directory of master tables (the delivery total rewards, resource boxes, honors, honor groups)") - hon.add_argument("--bundles", required=True, help="directory of decrypted honor and honor_frame packages") - hon.add_argument("--out-dir", required=True, help="runtime root; images land under honor/ and honor_frame/") - ipv = sub.add_parser("item-icons", - help="write index.json and textures///.png from " - "the mysekai item_preview and thumbnail material/item/tool packages, " - "keyed by the package path") - ipv.add_argument("--decrypted", required=True, - help="directory holding the decrypted mysekai item icon packages") - ipv.add_argument("--root", default=None, - help="runtime root whose mysekai-materials/items/fixtures/tools documents " - "give the coverage report") - ipv.add_argument("--out-dir", required=True, - help="output directory (mysekai-item-icons/ of a runtime root)") - a = sub.add_parser("fetch-apk", help="discover, download, or inspect an Android APK") - a.add_argument("--endpoint", default=None); a.add_argument("--timeout", type=float, default=30.0); a.add_argument("--retries", type=int, default=3) - asub = a.add_subparsers(dest="apk_command", required=True) - asub.add_parser("latest") - ad = asub.add_parser("download"); ad.add_argument("destination"); ad.add_argument("--url"); ad.add_argument("--sha256") - ai = asub.add_parser("inspect"); ai.add_argument("apk") - rp = sub.add_parser( - "player-render-pipeline", - help="extract the player-shipped MYSEKAI renderer pipeline and feature settings", - ) - rp.add_argument( - "--player-data", required=True, - help="extracted assets/bin/Data root containing globalgamemanagers", - ) - rp.add_argument( - "--types", required=True, - help="decoded managed assemblies for the same player build", - ) - rp.add_argument( - "--asset-pack", required=True, - help="APK/zip containing serialized resource files referenced by ResourceManager", - ) - rp.add_argument("--game-version", help="caller-known game version recorded as provenance") - rp.add_argument("--out", required=True, help="output JSON evidence path") - fal = sub.add_parser("fixture-actors", help="export all source-qualified character furniture animations") - fal.add_argument("--assets", required=True); fal.add_argument("--master", required=True) - fal.add_argument("--bundles", required=True); fal.add_argument("--workers", type=int, default=2) - fal.add_argument("--plan", action="store_true", help="write and summarize the source dependency plan without exporting") - fal.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="rotation key storage (default: snorm16); float32 disables lossy encoding") - fal.add_argument("--relation", action="append", choices=("fixture", "cut_scene"), default=None, - help="export only these relations' libraries (repeatable; default: all)") - ptc = sub.add_parser("player-timeline-clips", - help="export each player fixture timeline's CharacterAnimator clips against the " - "player avatar rig, one clip file per timeline package") - ptc.add_argument("--assets", required=True, - help="asset root holding fixture-timeline/ and the player timeline rows") - ptc.add_argument("--bundles", required=True, - help="directory of decrypted bundles (the avatar model and the fixture models)") - ptc.add_argument("--plan", action="store_true", help="summarize the clip plan without exporting") - stl = sub.add_parser("site-timelines", - help="export a site package's timeline tables, its step items (each prefab its " - "own glb with its prop clips) and its scene directors' rigs") - stl.add_argument("--assets", required=True, help="asset root receiving site-timeline/") - stl.add_argument("--bundles", required=True, - help="directory of decrypted bundles (the site packages and their dependencies)") - stl.add_argument("--package", action="append", required=True, - help="site package to export, e.g. mysekai__site__field__festivalgarden; repeatable") - fc = sub.add_parser("fixture-coverage", help="join every fixture master ID to extracted coverage") - fc.add_argument("--data-root", default=DEFAULT_DATA_ROOT); fc.add_argument("--out") - fc.add_argument("--region"); fc.add_argument("--source-version"); fc.add_argument("--json", action="store_true") - jp = sub.add_parser("jp", help="discover and download Japanese-server original bundles") - jsub = jp.add_subparsers(dest="jp_command", required=True) - jd = jsub.add_parser("discover", help="compare bootstrap metadata with the live official version list") - jd.add_argument("--bootstrap-url", default=None); jd.add_argument("--timeout", type=float, default=30.0) - jd.add_argument("--retries", type=int, default=3) - jm = jsub.add_parser("manifest", help="create a versioned JP bundle descriptor") - jm.add_argument("--app-version"); jm.add_argument("--app-hash") - jm.add_argument("--asset-version"); jm.add_argument("--asset-hash") - jm.add_argument("--versions-url", default=None) - jm.add_argument("--guest", action="store_true", help="explicitly create a guest only when public asset identity is unavailable") - jm.add_argument("--session", help="private guest-session JSON path; required with --guest") - jm.add_argument("--out", required=True); jm.add_argument("--timeout", type=float, default=30.0) - jm.add_argument("--retries", type=int, default=3) - jm.add_argument("--json", action="store_true", help="print the full descriptor as well as writing --out") - jpl = jsub.add_parser("plan", help="dependency-closed dry run from an online or offline descriptor") - jpl.add_argument("--manifest", required=True); jpl.add_argument("--family", action="append", default=[]) - jpl.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") - jpl.add_argument("--json", action="store_true", help="print root and bundle lists, not only counts") - jrt = jsub.add_parser("prepare", help="assemble a region-qualified Moly runtime in a separate staging directory") - jrt.add_argument("--manifest", required=True); jrt.add_argument("--workspace", required=True) - jrt.add_argument("--out", required=True); jrt.add_argument("--ui-from", required=True) - jrt.add_argument("--ui-jp-from", required=True, - help="JP UI extraction (ui-layout --region jp); its documents replace their donor copies") - jrt.add_argument("--vgmstream"); jrt.add_argument("--ffmpeg") - jrt.add_argument("--master-from", help="completed source-pinned master directory; defaults to Haruki when available") - jrt.add_argument("--actor-workers", type=int, default=2) - jrt.add_argument("--rotation-encoding", choices=ROTATION_ENCODINGS, default=DEFAULT_ROTATION_ENCODING, - help="motion rotation key storage (default: snorm16); float32 disables lossy encoding") - jms = jsub.add_parser("master", help="download an immutable, bundle-matched JP master snapshot") - jms.add_argument("--manifest", required=True); jms.add_argument("--out", required=True) - jms.add_argument("--revision", default="main"); jms.add_argument("--workers", type=int, default=6) - jms.add_argument("--provider", choices=("haruki", "sekai-world"), default="haruki") - jms.add_argument("--all-tables", action="store_true", help="fetch all master tables, not only Moly dependencies") - jdl = jsub.add_parser("download", help="download and verify original JP bundles without publishing an extracted release") - jdl.add_argument("--manifest", required=True); jdl.add_argument("--out") - jdl.add_argument("--workers", type=int, default=8); jdl.add_argument("--retries", type=int, default=4) - jdl.add_argument("--root", action="append", default=[]) - jdl.add_argument("--family", action="append", default=[]) - jpu = jsub.add_parser("pull", help="download, decrypt and extract a JP descriptor") - jpu.add_argument("--manifest", required=True); jpu.add_argument("--out") - jpu.add_argument("--extract-out", default="local-data-jp") - jpu.add_argument("--workers", type=int, default=4); jpu.add_argument("--retries", type=int, default=4) - jpu.add_argument("--master"); jpu.add_argument("--master-cache") - jpu.add_argument("--root", action="append", default=[], help="exact root bundle; repeatable") - s = sub.add_parser("shader", help="what a package's shaders declare and contain") - s.add_argument("package", help="one AssetBundle file") - s.add_argument("--platform", type=int, default=None, - help="count only this ShaderCompilerPlatform (9 is GLSL text)") - s.add_argument("--out", help="write the full census here as JSON") - s.add_argument("--programs", metavar="DIR", - help="write each program body here, named by shader, platform and record") - s.add_argument("--json", action="store_true", help="print the counts as JSON") - - args = ap.parse_args(argv); _configure_unity(args.unity_version) - if args.cmd == "fixture-actors": - from chara.fixture_actor_libraries import build_plan, export_fixture_actor_libraries - if args.plan: - report = build_plan(args.assets, args.master, args.bundles, rotation_encoding=args.rotation_encoding) - print(json.dumps({"summary": report["summary"], "gaps": report["gaps"]}, ensure_ascii=False)) - return 1 if report["gaps"] else 0 - report = export_fixture_actor_libraries(args.assets, args.master, args.bundles, - workers=args.workers, unity_version=args.unity_version or "2022.3.62f3", - rotation_encoding=args.rotation_encoding, relations=args.relation, - progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) - print(json.dumps({key: value for key, value in report.items() if key != "results"}, ensure_ascii=False)) - return 1 if report["failed"] else 0 - if args.cmd == "player-timeline-clips": - from chara.player_timeline_clips import build_plan, export_player_timeline_clips - if args.plan: - plan = build_plan(args.assets) - print(json.dumps({"summary": plan["summary"], "gaps": plan["gaps"]}, ensure_ascii=False)) - return 1 if plan["gaps"] else 0 - report = export_player_timeline_clips(args.assets, args.bundles, - progress=lambda row: print(json.dumps(row, ensure_ascii=False), file=sys.stderr)) - print(json.dumps({key: value for key, value in report.items() if key != "packages"}, - ensure_ascii=False)) - return 1 if report["gaps"] else 0 - if args.cmd == "site-timelines": - from sites.step_items import export_site_timelines - report = export_site_timelines(args.bundles, args.assets, list(args.package)) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "fixture-coverage": - from fixtures.coverage import write_coverage - path, report = write_coverage(args.data_root, args.out, region=args.region, - source_version=args.source_version) - print(json.dumps(report if args.json else {**report["summary"], "report": str(path)}, ensure_ascii=False)) - return 1 if report["summary"]["runtime"]["failed"] else 0 - if args.cmd == "jp": - from . import jp as jp_assets - try: - if args.jp_command == "discover": - options = {"timeout": args.timeout, "retries": args.retries} - if args.bootstrap_url: options["bootstrap_url"] = args.bootstrap_url - report = jp_assets.discover(**options) - elif args.jp_command == "manifest": - report = jp_assets.fetch_manifest(args.out, app_version=args.app_version, - app_hash=args.app_hash, session_file=args.session, - asset_version=args.asset_version, - asset_hash=args.asset_hash, guest=args.guest, - **({"versions_url": args.versions_url} - if args.versions_url else {}), - timeout=args.timeout, retries=args.retries) - elif args.jp_command == "plan": - report = jp_assets.plan(args.manifest, families=args.family, include=args.root) - elif args.jp_command == "prepare": - from .runtime_pack import prepare_runtime - report = prepare_runtime(args.manifest, args.workspace, args.out, ui_from=args.ui_from, - ui_jp_from=args.ui_jp_from, - decoder=args.vgmstream, transcoder=args.ffmpeg, - master_from=args.master_from, actor_workers=args.actor_workers, - rotation_encoding=args.rotation_encoding, - progress=lambda label: print(label, file=sys.stderr)) - elif args.jp_command == "master": - from .jp_master import download_master_snapshot - report = download_master_snapshot(args.manifest, args.out, revision=args.revision, workers=args.workers, provider=args.provider, all_tables=args.all_tables) - report = {"identity": report["identity"], "tableCount": report["tableCount"], "out": args.out} - elif args.jp_command == "download": - from .jp_download import download_descriptor - report = download_descriptor(args.manifest, out=args.out, workers=args.workers, - retries=args.retries, include=args.root, families=args.family, - progress=lambda row: print(f"{row['status']}: {row['completed']}/{row['requested']} verified, {row['failed']} failed", file=sys.stderr)) - else: - report = jp_assets.pull_descriptor(args.manifest, out=args.out, - extract_out=args.extract_out, workers=args.workers, - retries=args.retries, master=args.master, - master_cache=args.master_cache, - include=args.root) - if args.jp_command == "manifest" and not args.json: - shown = {"source": report["source"], "bundles": len(report["bundles"]), "descriptor": args.out} - elif args.jp_command == "plan" and not args.json: - shown = {"source": report["source"], **report["summary"]} - else: - shown = report - print(json.dumps(shown, ensure_ascii=False)); return 0 - except (jp_assets.JPError, OSError, ValueError, KeyError) as exc: - print(f"error: {exc}", file=sys.stderr); return 1 - if args.cmd == "pull": - from .fetch import pull - source, cache = _master_source(args) - report = pull(args.manifest, args.out, args.asset_base_url, args.workers, args.retries, - master=source, master_cache=cache, extract_out=args.extract_out, - vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) - if args.json: - print(json.dumps(report, ensure_ascii=False)) - else: - print(f"downloaded {report['downloads']}/{report['requiredBundles']} bundles") - # The sound packages are the one part of the set that master rows name, - # so "no audio" has a reason the caller must see without asking for JSON. - audio = report.get("audio") or {} - if audio.get("roots"): - print(f"audio: {len(audio['roots'])} sound packages named by master tables") - elif audio.get("error"): - print(f"audio: no sound package pulled ({audio['error']})") - if audio.get("notInManifest"): - print(f"audio: {len(audio['notInManifest'])} named sound packages " - f"are not in this manifest: " - f"{', '.join(audio['notInManifest'])}") - _print_extract_summary(report["extraction"], args.extract_out or DEFAULT_DATA_ROOT) - return 1 if _extract_failed(report["extraction"]) else 0 - if args.cmd == "shader": - import UnityPy - from shaders import census as shader_census - entries = shader_census.census(UnityPy.load(args.package), - keep_code=bool(args.programs)) - counts = shader_census.totals(entries, args.platform) - if args.out: - from .jsonio import write_json - write_json(args.out, {"package": args.package, - "shaders": shader_census.without_code(entries)}) - if args.programs: - written = _write_programs(args.programs, entries, args.platform) - counts["bodiesWritten"] = written - if args.json: - print(json.dumps(counts, ensure_ascii=False)) - else: - scope = "all platforms" if args.platform is None else f"platform {args.platform}" - print(f"{counts['shaders']} shaders, {counts['records']} program records, " - f"{counts['uniquePrograms']} distinct programs ({scope})") - if counts["platformErrors"]: - print(f"{counts['platformErrors']} platform blob(s) failed to parse") - for name, records, unique in shader_census.by_shader(entries, args.platform)[:10]: - print(f" {unique:6d} distinct / {records:6d} records {name}") - return 1 if counts["platformErrors"] else 0 - if args.cmd == "ui-audio": - from ui.audio import extract_ui_audio - report = extract_ui_audio(args.apk, args.out, decoder=args.vgmstream, transcoder=args.ffmpeg) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "fetch-apk": - from . import apk - forwarded = [] - if args.endpoint: forwarded += ["--endpoint", args.endpoint] - forwarded += ["--timeout", str(args.timeout), "--retries", str(args.retries), args.apk_command] - if args.apk_command == "download": - forwarded += [args.destination] - if args.url: forwarded += ["--url", args.url] - if args.sha256: forwarded += ["--sha256", args.sha256] - elif args.apk_command == "inspect": - forwarded += [args.apk] - return apk.main(forwarded) - if args.cmd == "player-render-pipeline": - from .player_render_pipeline import ( - extract_mysekai_render_pipeline, - write_mysekai_render_pipeline, - ) - try: - document = extract_mysekai_render_pipeline( - args.player_data, - args.types, - args.asset_pack, - unity_version=args.unity_version, - game_version=args.game_version, - ) - write_mysekai_render_pipeline(args.out, document) - except (OSError, ValueError, KeyError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - first = document["rendererSlots"][0]["objectKey"] if document["rendererSlots"] else None - count = len(document["renderers"].get(first, {}).get("features", [])) if first else 0 - print(json.dumps({ - "unityVersion": document["unityVersion"], - "gameVersion": document["gameVersion"], - "rendererSlots": len(document["rendererSlots"]), - "primaryRendererFeatures": count, - "out": args.out, - }, ensure_ascii=False)) - return 0 - if args.cmd == "extract": - from .extract import extract_manifest - from .assets.packages import builtin_archive_paths - source, cache = _master_source(args) - report = extract_manifest(args.manifest, args.bundles, args.out, args.unity_version, - master=source, master_cache=cache, - player_data=args.player_data, - player_types=args.player_types, - player_declarations=args.player_declarations, - fixture_meshes=args.fixture_meshes, - fixture_particles=args.fixture_particles, - rotation_encoding=args.rotation_encoding, - shader_compiler=args.shader_compiler, - builtin_resources=builtin_archive_paths( - args.builtin_resources), - vgmstream=args.vgmstream, ffmpeg=args.ffmpeg) - if args.json: - print(json.dumps(report, ensure_ascii=False)) - else: - if report.get("discovery"): - d = report["discovery"] - print(f"discovered {d['selected']} pack bundles under {args.bundles} " - f"(ignored {d['ignored']} unrelated files)") - _print_extract_summary(report, args.out) - return 1 if _extract_failed(report) else 0 - from chara import characters - if args.cmd == "motion-library": - from chara.motion_library import export_motion_library - names = [s for s in args.clips.split(",") if s] if args.clips else None - report = export_motion_library(args.reference_bundle, args.motion_bundle, args.out_dir, args.name, - aux=tuple(args.aux_bundle), names=names, - rotation_encoding=args.rotation_encoding, - controller_bundles=args.controller_bundle) - print(json.dumps(report, ensure_ascii=False)); return 0 - if args.cmd == "registry": - source, cache = _master_source(args) - if not source: - ap.error("registry needs master tables: pass --master or --master-url") - from .assets.manifest import parse_manifest - from .assets.router import route - from .extract import _unit_id - from .master import Master, MissingTable - from chara.registry import build_registry - names = parse_manifest(args.manifest) - units = sorted({u for n in names if route(n) and route(n).domain == "character" - for u in [_unit_id(n)] if u is not None}) - index = None - if args.motion_index: - with open(args.motion_index, encoding="utf-8") as handle: - index = json.load(handle) - source, cache = _master_source(args) - master = Master(source, cache_dir=cache) - try: - client_configs = master.client_configs() - except MissingTable: - client_configs = None - document = build_registry(master, units, index, client_configs=client_configs) - with open(args.out, "w", encoding="utf-8", newline="\n") as handle: - json.dump(document, handle, ensure_ascii=False, indent=1, allow_nan=False) - handle.write("\n") - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - - if args.cmd == "talks": - source, cache = _master_source(args) - if not source: - ap.error("talks needs master tables: pass --master or --master-url") - from chara.talks import extract_talks - source, cache = _master_source(args) - print(json.dumps(extract_talks(source, args.bundle, args.out, - master_cache=cache, - lib_bundle=args.lib), ensure_ascii=False)) - return 0 - - if args.cmd == "tweets": - source, cache = _master_source(args) - if not source: - ap.error("tweets needs master tables: pass --master or --master-url") - from chara.tweets import extract_tweets - print(json.dumps(extract_tweets(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "tweet-tables": - source, cache = _master_source(args) - if not source: - ap.error("tweet-tables needs master tables: " - "pass --master or --master-url") - from chara.tweet_tables import extract_tweet_tables - print(json.dumps(extract_tweet_tables(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "site-groups": - source, cache = _master_source(args) - if not source: - ap.error("site-groups needs master tables: " - "pass --master or --master-url") - from sites.site_groups import extract_site_groups - print(json.dumps(extract_site_groups(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "ui-action-icons": - from ui.action_icon import export_action_icons - print(json.dumps(export_action_icons(args.bundle, args.out_dir), - ensure_ascii=False)) - return 0 - - if args.cmd == "browser-assets": - from ui.character_portrait import export_character_portraits - result = export_character_portraits(args.player_data, args.asset_root, - source_region=args.source_region, - source_game_version=args.source_game_version) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 - - if args.cmd == "fixture-presentation": - from fixtures.presentation import extract_custom_geometry, enrich_presentation - extracted = extract_custom_geometry(args.bundles, Path(args.asset_root) / "custom-fixture-models") - if extracted.get("failed"): - print(json.dumps({"customGeometry": extracted}, ensure_ascii=False)) - return 1 - print(json.dumps(enrich_presentation(args.master, args.asset_root), ensure_ascii=False)) - return 0 - - if args.cmd == "validate-runtime": - from .runtime_validation import validate_runtime - report = validate_runtime(args.asset_root, verify_actor_hashes=args.verify_actor_hashes) - print(json.dumps(report, ensure_ascii=False)) - return 1 if report["missingResources"] else 0 - - if args.cmd == "fixture-animations": - from fixtures.animation_refresh import plan_animation_refresh, refresh_fixture_animations - report = plan_animation_refresh(args.asset_root) if args.plan else refresh_fixture_animations( - args.asset_root, args.bundles, packages=args.package, - progress=lambda row: print(json.dumps(row), file=sys.stderr)) - print(json.dumps(report, ensure_ascii=False)) - return 0 - - if args.cmd == "timeline-capabilities": - from fixtures.timeline_capabilities import build_capabilities - report = build_capabilities(args.asset_root) - print(json.dumps({"prefabs": len(report["prefabs"]), "packages": len(report["sources"])})) - return 0 - - if args.cmd == "ui-layout" and args.region == "jp": - from ui.prefab_layout import extract_layout_jp - if not (args.apk and args.client_version and args.master and args.prefab): - ap.error("ui-layout --region jp needs --apk (base and asset pack), " - "--client-version, --master and at least one --prefab") - result = extract_layout_jp(args.apk, args.out_dir, client_version=args.client_version, - master_dir=args.master, prefabs=args.prefab, - bundles_root=args.bundles, - font_bundles_root=args.font_bundles) - for item in result["written"]: - summary = item["summary"] - print(f"layout: {item['family']}/{item['prefab']} " - f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " - f"texts={summary['textComponents']} " - f"partial={len(summary['partialComponents'])} " - f"unresolved={len(summary['unresolvedPointers'])}") - for item in result["failures"]: - print(f"layout FAILED {item['prefab']}: {item['reason']}") - print(f"manifest: {result['manifestPath']}") - return 0 if not result["failures"] else 1 - - if args.cmd == "ui-layout": - from ui.prefab_layout import extract_layout - if not args.player_data: - ap.error("ui-layout --region cn needs --player-data") - result = extract_layout(args.player_data, args.out_dir, - bundles_root=args.bundles, - bundle_manifest=args.bundle_manifest, - prefabs=args.prefab, - client_version=args.client_version) - for item in result["settings"]: - print(f"settings: {item['document']} from {item['serializedFile']}:{item['pathId']}") - if not result["settings"]: - print("settings: palette.json and tween-settings.json not written " - "(they name the client version: pass --client-version)") - for item in result["written"]: - summary = item["summary"] - print(f"layout: {item['family']}/{item['prefab']} " - f"nodes={summary['nodes']} sprites={summary['spriteReferences']} " - f"texts={summary['textComponents']} " - f"partial={len(summary['partialComponents'])}") - for item in result["failures"]: - print(f"layout FAILED {item['prefab']}: {item['reason']}") - print(f"census: {result['censusPath']}") - return 0 if not result["failures"] else 1 - - if args.cmd == "ui-wordings": - from ui.prefab_layout import extract_container_wordings - result = extract_container_wordings(args.player_data, args.master, args.out, args.client_version) - print(f"wordings: {result['rows']} entries ({result['builtinRows']} built-in, " - f"{result['masterRows']} master rows, {result['masterOverwroteBuiltin']} built-in " - f"overwritten, {result['masterValuesRewritten']} master values rewritten, " - f"{result['builtinRejectedLines']} rejected lines) from " - f"{result['serializedFile']}:{result['pathId']} -> {result['path']}") - return 0 - - if args.cmd == "fixture-reactions": - from fixtures.reactions import extract_fixture_reactions - print(json.dumps(extract_fixture_reactions(args.bundle, args.out), ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-end-motions": - from sites.birthday_end_motions import extract_birthday_end_motions - print(json.dumps(extract_birthday_end_motions(args.bundle, args.out), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-party-talks": - source, cache = _master_source(args) - if not source: - ap.error("birthday-party-talks needs master tables: " - "pass --master or --master-url") - from chara.talks import attach_birthday_talks - print(json.dumps(attach_birthday_talks(source, args.bundle, args.talks, - args.out, master_cache=cache, - lib_bundle=args.lib), - ensure_ascii=False)) - return 0 - - if args.cmd == "fixture-master-slice": - source, cache = _master_source(args) - if not source: - ap.error("fixture-master-slice needs master tables: " - "pass --master or --master-url") - from fixtures.master_slice import export_fixture_master_slice - print(json.dumps(export_fixture_master_slice(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-parties": - source, cache = _master_source(args) - if not source: - ap.error("birthday-parties needs master tables: " - "pass --master or --master-url") - from sites.birthday_parties import extract_birthday_parties - print(json.dumps(extract_birthday_parties(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "birthday-party-delivery": - source, cache = _master_source(args) - if not source: - ap.error("birthday-party-delivery needs master tables: " - "pass --master or --master-url") - from sites.birthday_party_delivery import extract_birthday_party_delivery - print(json.dumps(extract_birthday_party_delivery(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "gate-visit-policy": - source, cache = _master_source(args) - if not source: - ap.error("gate-visit-policy needs master tables: " - "pass --master or --master-url") - from sites.gate_visit_policy import extract_gate_visit_policy - print(json.dumps(extract_gate_visit_policy(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - if args.cmd == "time-manager": - from core.time_manager import extract_time_manager - document = extract_time_manager(args.source, args.out) - print(json.dumps({"source": document["source"], "timeManager": document["timeManager"]}, - ensure_ascii=False)) - return 0 - if args.cmd == "client-config": - source, cache = _master_source(args) - if not source: - ap.error("client-config needs master tables: " - "pass --master or --master-url") - from core.client_config import extract_client_config - print(json.dumps(extract_client_config(source, args.out, master_cache=cache), - ensure_ascii=False)) - return 0 - - if args.cmd == "master-tables": - source, cache = _master_source(args) - if not source: - ap.error("master-tables needs master tables: " - "pass --master or --master-url") - from core.master_tables import extract_master_tables - print(json.dumps(extract_master_tables(source, args.out, - master_cache=cache, tables=args.tables), - ensure_ascii=False)) - return 0 - - if args.cmd == "emoticons": - from chara.emoticons import extract_emoticons - print(json.dumps(extract_emoticons(args.bundle, args.out_dir), ensure_ascii=False)) - return 0 - if args.cmd == "avatar-parts": - from chara.avatar_parts import extract_avatar_parts - print(json.dumps(extract_avatar_parts(args.bundle, args.out_dir), ensure_ascii=False)) - return 0 - if args.cmd == "harvest-tools": - from chara.harvest_tools import extract_harvest_tools - result = extract_harvest_tools(args.bundle, args.out_dir, bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "player-tool-models": - from chara.harvest_tools import extract_player_tool_models - result = extract_player_tool_models(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "effect-resources": - from .resource_exports import export_effect_resources - print(json.dumps(export_effect_resources( - args.player_data, args.types, args.out, declarations=args.declarations, - bundle_root=args.bundle_root), ensure_ascii=False)) - return 0 - if args.cmd == "layout-grid": - from .layout_grid import export_layout_grid - print(json.dumps(export_layout_grid(args.player_data, args.types, args.out_dir), - ensure_ascii=False, indent=2)) - return 0 - if args.cmd == "drop-item-shade": - from .resource_exports import export_drop_item_shade - print(json.dumps(export_drop_item_shade(args.player_data, args.types, args.out), - ensure_ascii=False)) - return 0 - if args.cmd == "stencil-shadow": - from .resource_exports import export_stencil_shadow - print(json.dumps(export_stencil_shadow(args.player_data, args.types, args.out), - ensure_ascii=False)) - return 0 - if args.cmd == "resource-prefab": - from .resource_exports import export_resource_prefab - from .assets.packages import builtin_archive_paths - document = export_resource_prefab(args.player_data, args.types, - args.load_name, args.out_dir, - builtin_archive_paths(args.builtin_resources)) - print(json.dumps({"file": document["file"], "objects": document["objects"], - "roots": [root["name"] for root in document["roots"]], - "particles": len(document["particles"]), - "unsupported": len(document["unsupported"])}, - ensure_ascii=False)) - return 0 - if args.cmd == "resource-programs": - from .resource_exports import export_resource_programs - document = export_resource_programs(args.player_data, args.types, args.load_name, - args.out_dir, args.shader_compiler) - print(json.dumps({"file": document["file"], - "shaders": [(row["name"], row["state"], len(row["materials"])) - for row in document["shaders"]]}, ensure_ascii=False)) - return 0 - if args.cmd == "fixture-particles": - from .assets.packages import PackageStore, builtin_archive_paths, require_paths - from fixtures.particles import DOMAINS, extract_from_store - require_paths(args.bundle) - builtins = builtin_archive_paths(args.builtin_resources) - require_paths(builtins, "built-in resource") - store = PackageStore(list(args.bundle) + builtins, args.bundle_root) - result = extract_from_store(store, args.out_dir, - shader_compiler=args.shader_compiler, - domains=tuple(args.domain or DOMAINS)) - print(json.dumps(result, ensure_ascii=False)) - return 0 - if args.cmd == "prefab-package": - from sites.prefab_package import extract_prefab_package - print(json.dumps(extract_prefab_package(args.bundle, args.prefab, args.out_dir, - bundle_root=args.bundle_root), - ensure_ascii=False)) - return 0 - if args.cmd == "site-sound-wave": - from sites.sound_wave import extract_site_sound_wave - result = extract_site_sound_wave(args.bundle, args.out, bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if result["failed"] else 0 - if args.cmd == "site-common-action-effects": - from sites.action_effects import extract_site_common_action_effects - result = extract_site_common_action_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-harvest-object-effects": - from sites.action_effects import extract_site_harvest_object_effects - result = extract_site_harvest_object_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-action-effects": - from sites.action_effects import extract_site_action_effects - result = extract_site_action_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "site-home-effects": - from sites.action_effects import extract_site_home_effects - result = extract_site_home_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "fixture-put-effects": - from sites.action_effects import extract_fixture_put_effects - result = extract_fixture_put_effects(args.bundle, args.out_dir, - bundle_root=args.bundle_root) - print(json.dumps(result, ensure_ascii=False)) - return 1 if any(row["status"] == "failed" for row in result["perBundle"].values()) else 0 - if args.cmd == "player-avatar": - from chara.player_avatar import export_player_avatar, write_motion_manifest - record = export_player_avatar(args.bundle, args.out_dir, name=args.name) - manifest_path = str(Path(args.out_dir) / f"{args.name}.motion-manifest.json") - write_motion_manifest(record, manifest_path) - print(json.dumps({ - "glb": record["glb"], "index": record["index"], - "motionManifest": manifest_path, - "sourcePackages": record["sourcePackages"], - "playerMesh": record["playerMesh"], - "counts": record["counts"]}, ensure_ascii=False)) - return 0 - if args.cmd == "phenomena": - from .assets.packages import builtin_archive_paths, require_paths - from phenomena.environments import extract_phenomena - source, cache = _master_source(args) - # --bundle takes paths, not bare package names: PackageStore keys the - # store by basename and UnityPy.load answers a path that is not there - # with an empty environment and no error, so a bare name runs green - # over zero objects. Refuse it here, before any work is done. - require_paths(args.bundle) - require_paths(args.site_bundle) - report = extract_phenomena(args.bundle, args.out_dir, - bundle_root=args.bundle_root, master=source, - master_cache=cache, vgmstream=args.vgmstream, - ffmpeg=args.ffmpeg, - shader_compiler=args.shader_compiler, - extra_archives=builtin_archive_paths( - args.builtin_resources), - site_bundles=args.site_bundle) - print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, - ensure_ascii=False)) - return 0 - if args.cmd == "mysekai-audio": - from phenomena.audio_corpus import extract_audio_corpus - report = extract_audio_corpus(args.talks, args.manifest, - args.bundle_root, args.out_dir, - decoder=args.vgmstream, - transcoder=args.ffmpeg) - print(json.dumps({k: v for k, v in report.items() if k != "audio"}, - ensure_ascii=False)) - return 0 - if args.cmd == "cutscene-audio": - from phenomena.cutscene_audio import extract_cutscene_audio - report = extract_cutscene_audio(args.clips, args.cut_scenes, args.manifest, - args.bundle_root, args.out_dir, - decoder=args.vgmstream, transcoder=args.ffmpeg) - print(json.dumps(report, ensure_ascii=False)) - return 0 - if args.cmd == "audio-sequences": - if Path(args.out).resolve() == Path(args.loop).resolve(): - ap.error("audio-sequences writes a copy: --out must differ from --loop") - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.audio_sequences import annotate_loop - report = annotate_loop(args.loop, args.bundle_root, args.out) - print(json.dumps({"counts": report["counts"], - "structuredByType": report["structuredByType"], - "archiveCuesByReferenceType": report["archiveCuesByReferenceType"], - "archiveSequencesByType": report["archiveSequencesByType"], - "answeredByTree": {k: len(v) for k, v - in report["answeredByTree"].items()}, - "failures": len(report["failures"]), - "disagreements": len(report["disagreements"]), - "sampleDifferences": len(report["sampleDifferences"])}, - ensure_ascii=False)) - return 0 - if args.cmd == "record-audio": - source, cache = _master_source(args) - if not source: - ap.error("record-audio needs master tables: " - "pass --master or --master-url") - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.record_audio import extract_record_audio - document = extract_record_audio(source, args.bundle_root, args.out, - bundle_version=args.bundle_version, - decoder=args.vgmstream, master_cache=cache, - only=args.only) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-storage": - from phenomena.record_audio import attach_storage - document = attach_storage(args.input, args.out, server=args.server, - workers=args.workers) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-carry": - source, cache = _master_source(args) - if not source: - ap.error("record-audio-carry needs the target master tables: " - "pass --master or --master-url") - from phenomena.record_audio import carry_record_audio - document = carry_record_audio(source, args.manifest, args.source, - args.source_manifest, args.out, - region=args.region, master_cache=cache) - print(json.dumps(document["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-fetch": - from .unity import configure_fallback_unity_version - configure_fallback_unity_version(getattr(args, "unity_version", None)) - from phenomena.record_audio import fetch_record_bundles - receipt = fetch_record_bundles(args.manifest, args.document, args.asset_base_url, - args.out, workers=args.workers) - print(json.dumps(receipt["summary"], ensure_ascii=False)) - return 0 - if args.cmd == "record-audio-merge": - from phenomena.record_audio import merge_record_audio - print(json.dumps(merge_record_audio(args.inputs, args.out)["summary"], - ensure_ascii=False)) - return 0 - if args.cmd == "audio-categories": - from phenomena.audio_categories import write_category_table - document = write_category_table(args.configuration, args.out) - print(json.dumps({"source": document["source"], - "categories": [(row["name"], row["index"], row["volume"]) - for row in document["categories"]]}, - ensure_ascii=False)) - return 0 - if args.cmd == "partvoice-routes": - source, cache = _master_source(args) - if not source: - ap.error("partvoice-routes needs master tables: " - "pass --master or --master-url") - from .master import Master - from phenomena.partvoice import (build_partvoice_routes, - route_counts, write_partvoice_routes) - roster = json.loads(Path(args.roster).read_text(encoding="utf-8")) - document = build_partvoice_routes(Master(source, cache_dir=cache), - roster) - path = write_partvoice_routes(document, args.out_dir) - print(json.dumps({**route_counts(document), "file": str(path)}, - ensure_ascii=False)) - return 0 - if args.cmd == "material-thumbnails": - from ui.material_thumbnail import export_material_thumbnails - from .extract import DEFAULT_UNITY_VERSION - result = export_material_thumbnails( - args.bundle, Path(args.master) / "materials.json", args.out_dir, - unity_version=args.unity_version or DEFAULT_UNITY_VERSION) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 if not result["coverage"]["missingMaterialIds"] else 1 - if args.cmd == "honor-images": - from ui.honor_images import export_honor_images - from .extract import DEFAULT_UNITY_VERSION - result = export_honor_images(args.master, args.bundles, args.out_dir, - unity_version=args.unity_version or DEFAULT_UNITY_VERSION) - print(json.dumps({"counts": result["counts"], "bundles": len(result["bundles"]), - "missingBundles": result["missingBundles"]}, ensure_ascii=False)) - return 0 if not result["missingBundles"] else 1 - if args.cmd == "item-icons": - from ui.item_icon import export_item_icons, package_paths - packages = package_paths(args.decrypted) - masters = {} - if args.root: - for key, name in (("materials", "mysekai-materials.json"), - ("items", "mysekai-items.json"), - ("fixtures", "mysekai-fixtures.json"), - ("tools", "mysekai-tools.json")): - path = Path(args.root) / name - masters[key] = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None - result = export_item_icons(packages, args.out_dir, **masters) - print(json.dumps(result["coverage"], ensure_ascii=False)) - return 0 - if args.cmd == "sitemap-screen-layer": - from sites.screen_layer import extract_screen_layer_jp - result = extract_screen_layer_jp( - args.apk, Path(args.out_dir) / "site" / "sitemap" / "screen_layer", - client_version=args.client_version) - print(json.dumps({"counts": result["counts"], "path": result["path"]}, - ensure_ascii=False)) - return 0 - if args.cmd == "site": - from .assets.packages import require_paths - from sites.pack import extract_sites - source, cache = _master_source(args) - # Same refusal as the phenomena command: a bare package name under - # --bundle would extract zero objects and report green. - require_paths(args.bundle) - report = extract_sites(args.bundle, args.out_dir, - bundle_root=args.bundle_root, master=source, - master_cache=cache) - print(json.dumps({k: v for k, v in report.items() if k != "perBundle"}, - ensure_ascii=False)) - return 0 - if args.cmd == "alone-actions": - from chara.alone_actions import write_alone_actions - print(json.dumps(write_alone_actions(args.bundle, args.out), ensure_ascii=False)) - return 0 - if args.cmd == "facial-tables": - tables = characters.facial_tables(args.bundle) - with open(args.out, "w", encoding="utf-8", newline="\n") as fh: - json.dump(tables, fh, ensure_ascii=False, indent=1, allow_nan=False); fh.write("\n") - print(json.dumps({k: len(v) for k, v in tables.items()})); return 0 - sampled = None - if args.motion_bundle and args.clips: - sampled = characters.sample_clips(args.motion_bundle, [s for s in args.clips.split(",") if s]) - facial = None - if args.facial_tables: - with open(args.facial_tables, encoding="utf-8") as fh: facial = json.load(fh) - cw, ch = args.atlas_cell.lower().split("x") - report = characters.extract_character(args.bundle, args.out_dir, args.name, sampled=sampled, unit_id=args.unit_id, facial=facial, atlas_cell=(int(cw), int(ch)), with_cloth=not args.no_cloth, aux=tuple(args.aux_bundle)) - print(json.dumps(report, ensure_ascii=False)); return 0 - +from cli.main import main +from cli.legacy import _extract_failed, _print_extract_summary if __name__ == "__main__": sys.exit(main()) diff --git a/src/core/content.py b/src/core/content.py new file mode 100644 index 0000000..0dd5ad3 --- /dev/null +++ b/src/core/content.py @@ -0,0 +1,20 @@ +"""Canonical bytes and hashes for content-addressed contract documents. + +This is the existing pack v2 encoding. Domain writers keep their authored +nonfinite-number representation; this encoder accepts only finite JSON. +""" +import hashlib +import json + + +def canonical_bytes(document): + return (json.dumps(document, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False) + "\n").encode("utf-8") + + +def content_hash(data): + return hashlib.sha256(data).hexdigest() + + +def document_hash(document): + return content_hash(canonical_bytes(document)) diff --git a/src/core/extract.py b/src/core/extract.py index 58f3bc3..d789ff4 100644 --- a/src/core/extract.py +++ b/src/core/extract.py @@ -122,6 +122,7 @@ "motion": {"AnimationClip", "Avatar"}, "facial": {"MonoBehaviour"}, "performance": {"TextAsset"}, + "communication": {"TextAsset"}, "talk": {"TextAsset"}, "talk-constants": {"TextAsset"}, "camera": {"MonoBehaviour", "PlayableDirector"}, @@ -303,6 +304,7 @@ def _configure_unity_version(unity_version): "motion-library.index.json", "facial-tables.json", "alone-actions.json", + "communication-actions.json", "talks.json", "tweets.json", "emoticons/emoticons.json", @@ -1688,6 +1690,11 @@ def extract_manifest(manifest, bundles, out, unity_version=None, master=None, elif target.domain == "performance": from chara.alone_actions import write_alone_actions result = write_alone_actions(str(bundle), str(out / "alone-actions.json")) + elif target.domain == "communication": + if not master: + raise ValueError("communication actions require master tables") + from chara.communication_actions import extract_communication_actions + result = extract_communication_actions(master, str(bundle), str(out / "communication-actions.json"), master_cache) else: from chara.characters import facial_tables target_path = out / "facial-tables.json" diff --git a/src/fixtures/master_slice.py b/src/fixtures/master_slice.py index 20a1b3e..8eb53ca 100644 --- a/src/fixtures/master_slice.py +++ b/src/fixtures/master_slice.py @@ -108,6 +108,12 @@ def export_fixture_master_slice( sub_genre = row.get("mysekaiFixtureSubGenreId") if sub_genre is not None and type(sub_genre) is not int: raise ValueError(f"row id={row['id']!r}: sub genre id must be an integer or null") + costs = {field: row.get(field) for field in ("firstPutCost", "secondPutCost")} + for field, value in costs.items(): + # Older clients do not expose cost limits. Missing stays null; + # consumers cannot turn an absent cost into a free placement. + if value is not None and (type(value) is not int or value < 0): + raise ValueError(f"row id={row['id']!r}: {field} must be a nonnegative integer or null") grid = row["gridSize"] for axis in ("width", "depth", "height"): if axis not in grid: @@ -130,6 +136,7 @@ def export_fixture_master_slice( "gridDepth": int(grid["depth"]), "gridHeight": int(grid["height"]), "putSoundId": put_sound, + **costs, } ) out.sort(key=lambda record: record["id"]) diff --git a/src/fixtures/particles.py b/src/fixtures/particles.py index 55ac242..434b437 100644 --- a/src/fixtures/particles.py +++ b/src/fixtures/particles.py @@ -767,7 +767,8 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, domain the router gives them -- one package of a domain whose other packages are not read this way (the travel cannon of the site domain). A named package the store does not hold is not extracted; its document's - ``domain`` is still the one the router reports. + ``domain`` is the routed domain, or ``explicit`` when the router does not + claim that package. """ domains = tuple(domains) packages = tuple(sorted(set(packages))) @@ -786,7 +787,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, target_names, lookup_names = [], [] for name in sorted(names): target = route(name) - if target is not None and (target.domain in domains or name in packages): + if name in packages or target is not None and target.domain in domains: target_names.append(name) else: lookup_names.append(name) @@ -797,7 +798,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, documents = {} per_domain = {domain: 0 for domain in domains} for name in target_names: - per_domain.setdefault(route(name).domain, 0) + per_domain.setdefault(route(name).domain if route(name) else "explicit", 0) totals = {"packages": len(target_names), "lookupPackages": len(lookup_names), "emitters": 0, "materials": 0, "textures": 0, "unsupported": 0, "missing": 0, @@ -805,7 +806,7 @@ def extract_from_store(store, out_dir, *, shader_compiler=None, domains=DOMAINS, "systemReadFailures": 0, "rendererReadFailures": 0} for name in target_names: routed = route(name) - document, file_name = _export_package(store, name, out, routed.domain, + document, file_name = _export_package(store, name, out, routed.domain if routed else "explicit", lambda directory: _Images(directory), source.materials, source.geometry) for emitter in document["emitters"]: diff --git a/src/pack/gc.py b/src/pack/gc.py index fa57d24..16968c7 100644 --- a/src/pack/gc.py +++ b/src/pack/gc.py @@ -25,6 +25,21 @@ def catalog_garbage(output): """List unused blobs only after validating every retained release root.""" output = Path(output) + if not (output / "asset-packs.json").exists(): + from .task_results import retained_results + roots, blobs = retained_results(output) + if not roots: + raise ValueError("no release or extraction result roots; refusing unrooted collection") + on_disk = {p.relative_to(output / "blobs").as_posix(): p.stat().st_size + for p in (output / "blobs").rglob("*") if p.is_file()} + garbage = sorted(on_disk.keys() - blobs.keys()) + # Without a release selector, preserve manifests and channel/archive + # state. Such a store may be mid-migration rather than task-only. + if any((output / folder).exists() for folder in ("catalogs", "channels", "packages")): + raise ValueError("release objects exist without a default catalog; restore it before GC") + return {"delete": garbage, "delete_manifests": [], "delete_manifest_bytes": 0, + "channel_roots": {}, "roots": roots, "delete_bytes": sum(on_disk[p] for p in garbage), + "retained_catalogs": 0, "referenced_blobs": sorted(blobs)} errors, info = verify_catalog(output / "asset-packs.json", root=output) if errors: raise ValueError("cannot determine catalog garbage: " + "; ".join(errors)) diff --git a/src/pack/groups.py b/src/pack/groups.py index 963dbde..74b39f3 100644 --- a/src/pack/groups.py +++ b/src/pack/groups.py @@ -29,7 +29,7 @@ # checks it against the paths a browser run requested. ROOT_DOCUMENTS = ( "manifest.json", "source.json", "characters.json", "motion-library.glb", "motion-library.index.json", - "alone-actions.json", "facial-tables.json", "client-config.json", "birthday-parties.json", + "alone-actions.json", "communication-actions.json", "facial-tables.json", "client-config.json", "birthday-parties.json", "birthday-party-delivery.json", "mysekai-blueprints.json", "mysekai-fixtures.json", "mysekai-items.json", "mysekai-fixture-main-genres.json", "mysekai-fixture-sub-genres.json", @@ -61,12 +61,12 @@ "avatar", "avatar-parts", "camera", "cutscene-timeline", "emoticons", "fixture-areas", "fixture-attach", "fixture-interface", "fixture-meshes", "fixture-models", "fixture-gimmick", "fixture-particles-v2", "fixture-talks", "fixture-timeline", "perf-animations", - "phenomena", "site", "ui", "ui-layout-v2", "ui-jp", + "phenomena", "site", "ui", "ui-layout-v2", "ui-jp", "ui-bulkharvest", "field-camera", "house-transition", "live-transitioner", "effect-resources", "graphics-config", "layout-grid", "site-action-effects", "site-common-action-effects", "site-harvest-object-effects", "site-home-effects", "fixture-put-effects", "site-sound-wave", "site-timeline", - "honor", "honor_frame", "material-thumbnails", + "honor", "honor_frame", "material-thumbnails", "text", "tutorial", ) FORMATS = {".json", ".glb", ".gltf", ".png", ".jpg", ".jpeg", ".webp", ".ktx2", ".rgba8", ".ogg", ".wav", ".mp3", ".bin"} BUILD_EVIDENCE_FILES = {"export-report.json", "export-plan.json", "export-progress.json", diff --git a/src/pack/identity.py b/src/pack/identity.py index e298b69..0198fcf 100644 --- a/src/pack/identity.py +++ b/src/pack/identity.py @@ -7,7 +7,7 @@ strings, booleans and null (with arrays/objects); it never contains floats. """ from __future__ import annotations -import json +from core.content import canonical_bytes from .hashing import sha256_bytes MANIFEST_V2 = "moly-asset-manifest/2" @@ -17,11 +17,6 @@ MAX_SAFE_INTEGER = (1 << 53) - 1 -def canonical_bytes(document): - return (json.dumps(document, ensure_ascii=False, sort_keys=True, - separators=(",", ":"), allow_nan=False) + "\n").encode("utf-8") - - def content_projection(manifest): projection = {"schema": CONTENT_V1, "transforms": manifest["transforms"], "entries": [{key: entry[key] for key in ("path", "bytes", "content_sha256", "xf")} diff --git a/src/pack/task_results.py b/src/pack/task_results.py new file mode 100644 index 0000000..dc0ce7b --- /dev/null +++ b/src/pack/task_results.py @@ -0,0 +1,56 @@ +"""Extraction result roots in the existing pack store. + +The result is committed after its artifacts. Every retained result protects +its artifacts from release GC even before a product selects them. Results +hold no input paths, local clocks, credentials or mutable release labels. +""" +import re + +from core.content import canonical_bytes, content_hash, document_hash +from .paths import output_path +from .verify import strict_json + + +def read_result(root, key): + if not re.fullmatch(r"[0-9a-f]{64}", key): + raise ValueError("invalid task result key") + path = output_path(root, f"tasks/{key}.json") + if not path.exists(): + return None + raw = path.read_bytes() + result = strict_json(raw) + if (result.get("schema") != "moly-result/1" or result.get("key") != key + or document_hash(result["keyParts"]) != key or result.get("status") != "ok" + or canonical_bytes(result) != raw): + raise ValueError("invalid task result identity") + seen = set() + for artifact in result["artifacts"]: + logical = artifact["path"] + if (not isinstance(logical, str) or "\\" in logical or ":" in logical + or any(part in ("", ".", "..") for part in logical.split("/"))): + raise ValueError("invalid task artifact logical path") + output_path(root, logical) + if logical in seen: + raise ValueError("duplicate task artifact path") + seen.add(logical) + digest = artifact["sha256"] + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("invalid task artifact hash") + if artifact["blob"] != f"{digest[:2]}/{digest}.bin": + raise ValueError("invalid task artifact storage address") + data = output_path(root, "blobs/" + artifact["blob"]).read_bytes() + if type(artifact["size"]) is not int or len(data) != artifact["size"] or content_hash(data) != digest: + raise ValueError(f"corrupt task artifact: {logical}") + return result + + +def retained_results(root): + """Return verified roots and blobs; corrupt roots abort collection.""" + folder = output_path(root, "tasks") + roots, blobs = {}, {} + for path in sorted(folder.glob("*.json")): + result = read_result(root, path.stem) + roots[f"tasks/{path.name}"] = "extraction result" + for artifact in result["artifacts"]: + blobs[artifact["blob"]] = artifact["size"] + return roots, blobs diff --git a/src/pack/verify.py b/src/pack/verify.py index e935d7d..38d48d8 100644 --- a/src/pack/verify.py +++ b/src/pack/verify.py @@ -730,6 +730,16 @@ def include(path, label): default = root / "asset-packs.json" if default.is_file() and default.resolve() != catalog_path: include(default, "default") + # Extraction results and published catalogs share these blobs. A result + # must retain its referenced artifacts before a runtime release selects it. + from .task_results import retained_results + try: + task_roots, task_blobs = retained_results(root) + roots.update(task_roots) + referenced.update(task_blobs) + info["blob_sizes"].update(task_blobs) + except (OSError, ValueError, KeyError, TypeError) as exc: + errors.append(f"extraction results: {exc}") on_disk = {path.relative_to(root / "blobs").as_posix() for path in (root / "blobs").rglob("*") if path.is_file()} package_files = {path.relative_to(root).as_posix() diff --git a/src/perf/directors.py b/src/perf/directors.py index 66102a8..5e046eb 100644 --- a/src/perf/directors.py +++ b/src/perf/directors.py @@ -95,12 +95,13 @@ def _signal_receiver(store, record, component_id, owner, hierarchy): "hierarchy": hierarchy, "reactions": rows} -def _director(store, record, path_id, owner, hierarchy): +def _director(store, record, path_id, owner, hierarchy, transforms): tree = record.tree(path_id) return { "asset": identity(record, path_id), "gameObject": identity(*owner), "hierarchy": hierarchy, + "transforms": transforms, "timeline": reference(store, record, tree.get("m_PlayableAsset")), "enabled": tree.get("m_Enabled"), "updateMode": tree.get("m_DirectorUpdateMode"), @@ -214,7 +215,7 @@ def read_prefab_directors(store, package): component_kind = component_file.kinds[component_id] if component_kind == "PlayableDirector": prefab["directors"].append(_director( - store, component_file, component_id, owner, hierarchy)) + store, component_file, component_id, owner, hierarchy, own)) elif (component_kind == "MonoBehaviour" and component_file.script_of(component_id) in TIMELINE_VIEW_SCRIPTS): prefab["fixtureTimelineViews"].append(_fixture_timeline_view( diff --git a/src/perf/fixture_talks.py b/src/perf/fixture_talks.py index 4a55a33..b32c5a2 100644 --- a/src/perf/fixture_talks.py +++ b/src/perf/fixture_talks.py @@ -166,7 +166,7 @@ def classify_ops(text, script_name=None): def _text_assets(bundle): """``{TextAsset name: script text}``.""" out = {} - for obj in UnityPy.load(bundle).objects: + for obj in UnityPy.load(Path(bundle).read_bytes()).objects: if obj.type.name != "TextAsset": continue tree = obj.read_typetree() diff --git a/src/phenomena/audio.py b/src/phenomena/audio.py index e188a9b..e9ccae9 100644 --- a/src/phenomena/audio.py +++ b/src/phenomena/audio.py @@ -33,6 +33,8 @@ the random-without-repeat type. """ import re +import hashlib +import json import shutil import struct import subprocess @@ -163,13 +165,60 @@ def attach_sequences(entry, data): class Library: """Audio of one run: archives always, waveforms when the decoder is present.""" - def __init__(self, directory, prefix, decoder=None, transcoder=None): + def __init__(self, directory, prefix, decoder=None, transcoder=None, *, preserve_existing=False): self.root = Path(directory) self.prefix = prefix self.decoder = tool(DECODER, decoder) self.transcoder = tool(TRANSCODER, transcoder) self.packages = [] self.unsupported = [] + self.previous = None + if preserve_existing and (self.root / "loop.json").is_file(): + self.previous = json.loads((self.root / "loop.json").read_bytes()) + # Capture identity before add() can write an archive at its old path. + for entry in self.previous.get("packages", []): + if not entry.get("sourceSha256"): + archive = self._existing_file(entry.get("archive")) + if archive is not None: + entry["sourceSha256"] = hashlib.sha256(archive.read_bytes()).hexdigest() + + def _existing_file(self, logical): + if not isinstance(logical, str) or not logical.startswith(self.prefix + "/"): + return None + target = self.root / logical[len(self.prefix) + 1:] + if not target.resolve().is_relative_to(self.root.resolve()): + raise ValueError("existing audio path escaped output root") + return target if target.is_file() else None + + def _preserved_packages(self): + if self.previous is None: + return self.packages + current = {(row["package"], Path(row["archive"]).name): row for row in self.packages} + combined = [] + for prior in self.previous.get("packages", []): + key = (prior["package"], Path(prior["archive"]).name) + fresh = current.pop(key, None) + if fresh is None: + combined.append(prior) + continue + if not prior.get("sourceSha256") or prior["sourceSha256"] != fresh["sourceSha256"]: + raise ValueError("audio archive identity changed; use a fresh output root") + # A weather subset cannot erase the whole bank the corpus decoded. + keys = {(row.get("cue"), row.get("subsong")) for row in fresh["streams"]} + retained = [row for row in prior.get("streams", []) + if (row.get("cue"), row.get("subsong")) not in keys] + if any(not any(self._existing_file(row.get(kind)) is not None + for kind in ("wav", "ogg")) for row in retained): + raise ValueError("existing audio stream is missing; repair the corpus before extraction") + merged = {**fresh, "streams": [*retained, *fresh["streams"]]} + for name in ("archiveSetRevision", "archiveSetCount", "archiveSetIndex"): + if name in prior: + merged[name] = prior[name] + archive = self._existing_file(fresh["archive"]) + attach_sequences(merged, archive.read_bytes()) + combined.append(merged) + combined.extend(current.values()) + return combined @property def status(self): @@ -211,9 +260,15 @@ def add(self, package, name, data, cues=None, *, namespace=None, report_missing= prefix = f"{self.prefix}/{relative.as_posix()}" directory.mkdir(parents=True, exist_ok=True) archive = directory / f"{name}.acb" + source_sha = hashlib.sha256(data).hexdigest() + for prior in (self.previous or {}).get("packages", []): + if prior["package"] == package and Path(prior["archive"]).name == archive.name: + if prior.get("sourceSha256") != source_sha: + raise ValueError("audio archive identity changed; use a fresh output root") archive.write_bytes(data) entry = {"package": package, "archive": f"{prefix}/{archive.name}", - "archiveBytes": len(data), "status": self.status, "streams": []} + "archiveBytes": len(data), "sourceSha256": source_sha, + "status": self.status, "streams": []} if not self.decoder: entry["error"] = NO_DECODER self.packages.append(entry) @@ -256,13 +311,10 @@ def add(self, package, name, data, cues=None, *, namespace=None, report_missing= def finish(self): """Write the loop sidecar and return the index-level audio document. - The write is a wholesale replacement, not a merge: the file on disk - becomes exactly this run's package list. Merging with what a previous - run wrote lives on the corpus side (``audio_corpus._merge_loop``), - which is also what the corpus's resume reads as its skip roster — so - re-running the phenomena job in place over a directory the corpus - already wrote replaces that roster whole, and the next corpus run - re-decodes every package the phenomena run did not carry. + By default the index becomes exactly this run's package list. Weather + callers opt into preserve_existing so re-exporting its cue subset does + not erase the rest of the same-source sound corpus. Overlapping banks + must retain their exact archive hash; changing releases needs a new root. """ # Only tool names, never the paths they were found at: this document ships # next to the audio and would otherwise carry one machine's directory layout. @@ -270,7 +322,7 @@ def finish(self): "decoderPresent": bool(self.decoder), "transcoder": TRANSCODER, "transcoderPresent": bool(self.transcoder), - "packages": self.packages} + "packages": self._preserved_packages()} if not self.decoder: document["error"] = NO_DECODER elif not self.transcoder: diff --git a/src/phenomena/audio_corpus.py b/src/phenomena/audio_corpus.py index 2316f31..ed34ae0 100644 --- a/src/phenomena/audio_corpus.py +++ b/src/phenomena/audio_corpus.py @@ -122,6 +122,9 @@ def _corpus_talks(document): for unit in (document.get("units") or {}).values(): yield from unit.get("talks") or [] yield from document.get("talks") or [] + yield from (document.get("commonSomeCharacter") or {}).get("talks") or [] + for lua, row in ((document.get("tutorial") or {}).get("talks") or {}).items(): + yield {**row, 'lua': lua} def voice_requests(document): @@ -142,7 +145,7 @@ def voice_requests(document): # stripping a fixed number of underscore fields does not. stem = script if isinstance(script, str) and cue.startswith( CUE_PREFIX + script + "_") else voice_stem(cue) - requests.setdefault(TALK_VOICE + stem, set()).add(cue) + requests.setdefault(talk.get('voiceBank') or TALK_VOICE + stem, set()).add(cue) # A party script's cues live in the bank the talk engine loads under the # Birthday context, not in the plain one, and the talk document names that # bank per script; it is read from there rather than assumed from a prefix. diff --git a/src/phenomena/environments.py b/src/phenomena/environments.py index 0096090..da4f036 100644 --- a/src/phenomena/environments.py +++ b/src/phenomena/environments.py @@ -995,7 +995,7 @@ def extract_phenomena(bundles, out_dir, bundle_root=None, master=None, "schemaVersion": collision_document["schemaVersion"], **collision_document["summary"]} - library = Library(out / "audio", "audio", vgmstream, ffmpeg) + library = Library(out / "audio", "audio", vgmstream, ffmpeg, preserve_existing=True) sound_packages = {} wanted_cues = sorted({str(row.get("cue")) for row in sound_rows if row.get("cue")}) diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..ab35289 --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Versioned extraction tasks, reuse, and product assembly over the pack store.""" diff --git a/src/pipeline/contract.py b/src/pipeline/contract.py new file mode 100644 index 0000000..46cdeb1 --- /dev/null +++ b/src/pipeline/contract.py @@ -0,0 +1,44 @@ +"""Task descriptions separate content-affecting facts from source locations.""" +import re + +from core.content import document_hash + + +def key_parts(task): + return {"stage": task["stage"], "subject": task["subject"], + "params": task["params"], "atoms": task["atoms"], "context": task["context"], + "inputs": [{k: item[k] for k in ("role", "sha256", "size")} + for item in sorted(task["inputs"], key=lambda item: item["role"])]} + + +def validate(task): + if task.get("schema") != "moly-task/1": + raise ValueError("unsupported task schema") + if (not isinstance(task["subject"], str) or not task["subject"] + or not re.fullmatch(r"[a-z][a-z0-9_.-]*", task["stage"]["name"]) + or type(task["stage"]["version"]) is not int or task["stage"]["version"] < 1): + raise ValueError("invalid task stage or subject") + roles = set() + for item in task["inputs"]: + role = item["role"] + if not isinstance(role, str) or not role or role in roles: + raise ValueError("empty or duplicate task input role") + roles.add(role) + if (not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) + or type(item["size"]) is not int or item["size"] < 0 + or not isinstance(item.get("locators"), list)): + raise ValueError("invalid input identity or locators") + for field in ("params", "atoms", "context"): + if not isinstance(task[field], dict): + raise ValueError(f"{field} must be an object") + if task.get("key") != document_hash(key_parts(task)): + raise ValueError("task key does not match its declared inputs") + return task + + +def describe(stage, version, subject, inputs, *, params=None, atoms=None, context=None): + task = {"schema": "moly-task/1", "stage": {"name": stage, "version": version}, + "subject": subject, "inputs": sorted(inputs, key=lambda item: item["role"]), + "params": params or {}, "atoms": atoms or {}, "context": context or {}} + task["key"] = document_hash(key_parts(task)) + return validate(task) diff --git a/src/pipeline/depot.py b/src/pipeline/depot.py new file mode 100644 index 0000000..f72fabe --- /dev/null +++ b/src/pipeline/depot.py @@ -0,0 +1,86 @@ +"""Run versioned stages and retain their outputs in the existing pack CAS.""" +from pathlib import Path +import tempfile + +from core.atomic import exclusive_lock, write_bytes +from core.content import canonical_bytes, content_hash +from pack.paths import output_path +from pack.task_results import read_result, retained_results +from source.snapshot import read_inputs +from .contract import key_parts, validate + + +class Depot: + def __init__(self, root): + self.root = Path(root).resolve() + + def plan(self, task, *, previous=None): + from .stages import validate_installed + validate(task) + validate_installed(task) + result = read_result(self.root, task["key"]) + reasons = [] + if previous is not None: + validate(previous) + old, new = key_parts(previous), key_parts(task) + reasons = [field + "-changed" for field in new if old[field] != new[field]] + if not result and not reasons: + reasons = ["result-missing"] + return {"schema": "moly-plan/1", "task": task, + "status": "hit" if result else "run", "reasons": reasons or ["cached"]} + + def run(self, task, *, force=False): + from .stages import execute, validate_installed + validate(task) + validate_installed(task) + with exclusive_lock(self.root / ".publish.lock"): + previous = read_result(self.root, task["key"]) + if previous is not None and not force: + return {"status": "hit", "result": previous} + # Freeze and verify all bytes before calling a reader. A caller + # changing files during extraction cannot contaminate this task. + inputs = read_inputs(task["inputs"]) + with tempfile.TemporaryDirectory(prefix="moly-task-") as temp: + artifacts = execute(task, inputs, Path(temp)) + validate_installed(task) + rows = [] + for logical, data in sorted(artifacts.items()): + output_path(self.root, logical) + digest = content_hash(data) + blob = f"{digest[:2]}/{digest}.bin" + path = output_path(self.root, "blobs/" + blob) + if path.exists(): + if path.read_bytes() != data: + raise ValueError("corrupt artifact at content address") + else: + write_bytes(path, data) + rows.append({"path": logical, "sha256": digest, "size": len(data), "blob": blob}) + validate_installed(task) + result = {"schema": "moly-result/1", "key": task["key"], + "keyParts": key_parts(task), "status": "ok", "artifacts": rows} + if previous is not None and result != previous: + raise ValueError("stage changed output for the same task key; bump its version") + write_bytes(output_path(self.root, f"tasks/{task['key']}.json"), canonical_bytes(result)) + return {"status": "ran", "result": result} + + def materialize(self, key, destination): + destination = Path(destination).resolve() + if destination.is_relative_to(self.root) or self.root.is_relative_to(destination): + raise ValueError("layout and depot must be separate directories") + with exclusive_lock(self.root / ".publish.lock"): + result = read_result(self.root, key) + if result is None: + raise ValueError("task has no committed result") + # Discovery is installed last. New hashed documents are complete + # before an existing reader can see the new root index. + for artifact in sorted(result["artifacts"], key=lambda row: ( + row["path"] == "text/index.json" or row["path"].endswith("/latest.json"))): + write_bytes(output_path(destination, artifact["path"]), + output_path(self.root, "blobs/" + artifact["blob"]).read_bytes()) + # Do not hard-link editable layouts to immutable shared bytes. + return {"key": key, "files": len(result["artifacts"])} + + def verify(self): + with exclusive_lock(self.root / ".publish.lock"): + roots, blobs = retained_results(self.root) + return {"results": len(roots), "blobs": len(blobs), "bytes": sum(blobs.values())} diff --git a/src/pipeline/stages.py b/src/pipeline/stages.py new file mode 100644 index 0000000..46946fb --- /dev/null +++ b/src/pipeline/stages.py @@ -0,0 +1,171 @@ +"""Built-in stages call the same domain readers as direct exports. + +Only inputs consumed by the selected stage enter its key. Source-module +fingerprints also separate unreleased edits; versions remain the declared +semantic contract and installed dependency versions identify byte readers. +""" +from importlib.metadata import version +import json +from pathlib import Path + +from core.content import canonical_bytes, content_hash, document_hash +from source.snapshot import file_input +from .contract import describe + +VERSIONS = {"talks.extract": 1, "communication.extract": 1, "text.catalog": 1, "text.official": 1, "text.runtime": 1} +MODULES = { + "communication.extract": ("chara/communication_actions.py", "chara/alone_actions.py", "chara/talks.py", "core/master.py", "core/jsonio.py", "core/unity.py"), + "talks.extract": ("chara/talks.py", "chara/common_talks.py", "perf/fixture_talks.py", + "core/master.py", "core/assets/router.py", "core/unity.py"), + "text.catalog": ("chara/text_catalog.py", "core/jsonio.py"), + "text.official": ("chara/text_catalog.py",), + "text.runtime": ("pipeline/text_product.py",), +} +TALK_TABLES = ( + "mysekaiCharacterTalks", "mysekaiGameCharacterUnitGroups", + "mysekaiCharacterTalkConditions", "mysekaiCharacterTalkConditionGroups", + "mysekaiCharacterTalkPreActions", "mysekaiCharacterTalkTweets", +) + + +def atoms(stage): + root = Path(__file__).resolve().parents[1] + modules = (*MODULES[stage], "pipeline/stages.py", "core/content.py") + result = {"implementation": document_hash({name: content_hash((root / name).read_bytes()) + for name in modules})} + if stage in {"talks.extract", "communication.extract"}: + result["unitypy"] = version("UnityPy") + return result + + +def task(stage, subject, inputs, **params): + return describe(stage, VERSIONS[stage], subject, inputs, params=params, atoms=atoms(stage)) + + +def talk_task(master, bundle, lib, *, family="general", unity_version="2022.3.62f3"): + if family not in {"general", "fixture"}: + raise ValueError("unknown conversation family") + inputs = [file_input("bundle", bundle), file_input("lib", lib)] + tables = list(TALK_TABLES) + if family == "fixture": + tables += ["mysekaiCharacterTalkNoTalkMysekaiFixtureActions", "mysekaiFixtures"] + else: + # Optional tables are part of the input identity when present. Their + # absence is represented by the table list, not a made-up empty file. + tables += [name for name in ("birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", "mysekaiTutorialTalks") + if (Path(master) / (name + ".json")).is_file()] + inputs += [file_input("master/" + name, Path(master) / (name + ".json")) for name in tables] + return task("talks.extract", family, inputs, family=family, unityVersion=unity_version) + + +def catalog_task(corpora, *, region, source_revision, previous=None): + inputs = [file_input(f"corpus/{i:04d}", path) for i, path in enumerate(corpora)] + if previous: + inputs.append(file_input("previous", previous)) + return task("text.catalog", region, inputs, region=region, sourceRevision=source_revision) + + +def communication_task(master, bundle, *, unity_version="2022.3.62f3"): + return task("communication.extract", "characters", [file_input("bundle", bundle), + file_input("master/mysekaiCharacterTalkTogetherCommunications", Path(master) / "mysekaiCharacterTalkTogetherCommunications.json")], + unityVersion=unity_version) + + +def official_task(source, donor, *, locale): + return task("text.official", locale, + [file_input("source", source), file_input("donor", donor)], locale=locale) + + +def runtime_task(source, overlays=()): + inputs = [file_input("source", source)] + inputs += [file_input(f"overlay/{i:04d}", path) for i, path in enumerate(overlays)] + return task("text.runtime", "display", inputs) + + +def validate_installed(value): + name = value["stage"]["name"] + if name not in VERSIONS or value["stage"]["version"] != VERSIONS[name]: + raise ValueError("task stage/version is not installed") + if value["atoms"] != atoms(name): + raise ValueError("task implementation differs from this installation; regenerate the plan") + expected = {"talks.extract": {"family", "unityVersion"}, "communication.extract": {"unityVersion"}, + "text.catalog": {"region", "sourceRevision"}, + "text.official": {"locale"}, "text.runtime": set()} + if set(value["params"]) != expected[name]: + raise ValueError("task parameters differ from the stage contract") + roles = {item["role"] for item in value["inputs"]} + if name == "communication.extract": + if roles != {"bundle", "master/mysekaiCharacterTalkTogetherCommunications"}: + raise ValueError("communication requires bundle and source master") + elif name == "talks.extract": + if value["params"]["family"] not in {"general", "fixture"}: + raise ValueError("unknown conversation family") + permitted = set(TALK_TABLES) | {"birthdayPartyScenarios", "mysekaiCharacterTalkSomeCharacterTalks", "mysekaiTutorialTalks", + "mysekaiFixtures", "mysekaiCharacterTalkNoTalkMysekaiFixtureActions"} + if not {"bundle", "lib"} <= roles or any( + role not in {"bundle", "lib"} and role not in {"master/" + t for t in permitted} + for role in roles): + raise ValueError("unknown or missing talk input roles") + elif name == "text.official" and roles != {"source", "donor"}: + raise ValueError("official text requires source and donor") + elif name in {"text.catalog", "text.runtime"}: + prefix = "corpus/" if name == "text.catalog" else "overlay/" + singles = {"previous"} if name == "text.catalog" else {"source"} + if any(role not in singles and not (role.startswith(prefix) and role[len(prefix):].isdigit()) + for role in roles): + raise ValueError("unknown stage input role") + if (name == "text.runtime" and "source" not in roles + or name == "text.catalog" and not any(role.startswith(prefix) for role in roles)): + raise ValueError("stage has no source inputs") + + +def execute(value, inputs, workspace): + name, params = value["stage"]["name"], value["params"] + if name == "communication.extract": + from chara.communication_actions import extract_communication_actions, TABLE + from core.unity import configure_fallback_unity_version + configure_fallback_unity_version(params["unityVersion"]) + master = workspace / "master" + master.mkdir() + (master / (TABLE + ".json")).write_bytes(inputs["master/" + TABLE]) + bundle, out = workspace / "bundle", workspace / "communication-actions.json" + bundle.write_bytes(inputs["bundle"]) + extract_communication_actions(master, bundle, out) + return {"communication-actions.json": out.read_bytes()} + if name == "talks.extract": + from core.unity import configure_fallback_unity_version + from chara.talks import extract_talks + from perf.fixture_talks import extract_fixture_talks + configure_fallback_unity_version(params["unityVersion"]) + master = workspace / "master" + master.mkdir() + for role, data in inputs.items(): + if role.startswith("master/"): + (master / (role.removeprefix("master/") + ".json")).write_bytes(data) + bundle, lib, out = workspace / "bundle", workspace / "lib", workspace / "talks.json" + bundle.write_bytes(inputs["bundle"]) + lib.write_bytes(inputs["lib"]) + reader = extract_talks if params["family"] == "general" else extract_fixture_talks + reader(str(master), str(bundle), str(out), lib_bundle=str(lib)) + doc = json.loads(out.read_bytes()) + if params["family"] == "fixture": + summary = doc["summary"] + if summary.get("missingScripts") or summary.get("unknownOperations"): + raise ValueError("fixture conversation extraction is incomplete") + return {("talks.json" if params["family"] == "general" else "fixture-talks/talks.json"): + out.read_bytes()} + if name == "text.catalog": + from chara.text_catalog import build_catalog + doc = build_catalog([json.loads(data) for role, data in sorted(inputs.items()) if role.startswith("corpus/")], + region=params["region"], source_revision=params["sourceRevision"], + previous=json.loads(inputs["previous"]) if "previous" in inputs else None) + return {"text-source.json": canonical_bytes(doc)} + if name == "text.official": + from chara.text_catalog import match_official + doc = match_official(json.loads(inputs["source"]), json.loads(inputs["donor"]), locale=params["locale"]) + return {f"official-{params['locale']}.json": canonical_bytes(doc)} + if name == "text.runtime": + from .text_product import build_product + return build_product(json.loads(inputs["source"]), + [json.loads(data) for role, data in sorted(inputs.items()) if role.startswith("overlay/")]) + raise ValueError("unknown stage") diff --git a/src/pipeline/text_product.py b/src/pipeline/text_product.py new file mode 100644 index 0000000..c3d0606 --- /dev/null +++ b/src/pipeline/text_product.py @@ -0,0 +1,107 @@ +"""Small per-content display documents, with a bounded two-level index. + +The complete source registry stays in editorial storage. A player reads the +root, one routing bucket and the selected script's display/overlay documents. +Each script includes its linked tweets so a session fixes both together. +""" +from collections import defaultdict + +from core.content import canonical_bytes, content_hash, document_hash + +MAX_SHARD_BYTES = 2 * 1024 * 1024 + + +def build_product(source, overlays=()): + if source.get("schema") != "moly-text-source/1": + raise ValueError("unsupported text source schema") + scripts = {row["key"]: row for row in source["scripts"]} + if len(scripts) != len(source["scripts"]): + raise ValueError("duplicate text script identity") + all_lines = {line["id"]: line for row in scripts.values() for line in row["lines"]} + if len(all_lines) != sum(len(row["lines"]) for row in scripts.values()): + raise ValueError("duplicate text line identity") + locales = defaultdict(dict) + for overlay in overlays: + if (overlay.get("schema") != "moly-text-overlay/1" + or overlay.get("sourceRegion") != source["region"] + or overlay.get("sourceRevision") != source["revision"]): + raise ValueError("overlay does not belong to this source revision") + for entry in overlay["entries"]: + line = all_lines.get(entry["id"]) + if line is None or entry["sourceHash"] != line["sourceHash"]: + raise ValueError("overlay refers to a missing or changed source line") + if entry["id"] in locales[overlay["locale"]]: + raise ValueError("duplicate overlay line for the same locale") + locales[overlay["locale"]][entry["id"]] = entry + files, buckets, search = {}, defaultdict(dict), defaultdict(dict) + translations = {locale: defaultdict(dict) for locale in locales} + translated_search = {locale: defaultdict(dict) for locale in locales} + + def store(document, prefix="text"): + data = canonical_bytes(document) + if len(data) > MAX_SHARD_BYTES: + raise ValueError("text shard exceeds the runtime document limit") + digest = content_hash(data) + path = f"{prefix}/objects/{digest}.json" + files[path] = data + return {"path": f"objects/{digest}.json", "sha256": digest, "bytes": len(data)} + + for key, script in sorted(scripts.items()): + tweet_keys = {c["tweetKey"] for c in script["contents"] if c.get("tweetKey")} + if tweet_keys - scripts.keys(): + raise ValueError("missing linked tweet source") + selected = [script] + [scripts[k] for k in sorted(tweet_keys) if k != key] + display = {"schema": "moly-text-source/1", "region": source["region"], + "scripts": [{"key": row["key"], "lines": [ + {field: line[field] for field in ("id", "kind", "step", "text", "sourceHash")} + for line in row["lines"]]} for row in selected]} + display["revision"] = document_hash(display) + route = {"source": store(display)} + bucket = content_hash(key.encode("utf-8"))[:2] + search[bucket[0]][key] = [[line["id"], line["sourceHash"], line["kind"], line["text"]] + for line in script["lines"]] + ids = {line["id"] for row in selected for line in row["lines"]} + for locale, entries in sorted(locales.items()): + # Donor/global revision provenance stays in the source registry + # and root. A change to another conversation must not churn this + # immutable per-script payload. + selected_entries = [{field: entries[identity][field] + for field in ("id", "sourceHash", "text", "source")} + for identity in sorted(ids & entries.keys())] + if not selected_entries: + continue + doc = {"schema": "moly-text-overlay/1", "sourceRegion": source["region"], + "sourceRevision": display["revision"], "locale": locale, "entries": selected_entries} + doc["revision"] = document_hash(doc) + prefix = f"translations/{source['region']}/{locale}" + ref = store(doc, prefix) + translations[locale][bucket][key] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + own_ids = {line["id"] for line in script["lines"]} + own_entries = [[line["id"], line["sourceHash"], line["text"], line["source"]] + for line in selected_entries if line["id"] in own_ids] + if own_entries: + translated_search[locale][bucket[0]][key] = own_entries + buckets[bucket][key] = route + root = {"schema": "moly-text-runtime/1", "region": source["region"], + "sourceRevision": source["revision"], "routing": "sha256-utf8-prefix-2", + "buckets": {}, "searchBuckets": {}} + for bucket, routes in sorted(buckets.items()): + root["buckets"][bucket] = store({"schema": "moly-text-routes/1", "scripts": routes}) + for bucket, rows in sorted(search.items()): + root["searchBuckets"][bucket] = store({"schema": "moly-text-source-search/1", "scripts": rows}) + root["revision"] = document_hash(root) + files["text/index.json"] = canonical_bytes(root) + for locale, translation_buckets in sorted(translations.items()): + prefix = f"translations/{source['region']}/{locale}" + publication = {"schema": "moly-text-publication/1", "region": source["region"], + "locale": locale, "routing": "sha256-utf8-prefix-2", "buckets": {}, "searchBuckets": {}} + for bucket, routes in sorted(translation_buckets.items()): + ref = store({"schema": "moly-text-translation-routes/1", "scripts": routes}, prefix) + publication["buckets"][bucket] = {**ref, "path": ref["path"].removeprefix(prefix + "/")} + for bucket, rows in sorted(translated_search[locale].items()): + publication["searchBuckets"][bucket] = store({"schema": "moly-text-translation-search/1", "scripts": rows}, prefix) + publication["revision"] = document_hash(publication) + data = canonical_bytes(publication) + files[f"{prefix}/revisions/{publication['revision']}.json"] = data + files[f"{prefix}/latest.json"] = data + return files diff --git a/src/sites/bulk_harvest.py b/src/sites/bulk_harvest.py new file mode 100644 index 0000000..7ab2efa --- /dev/null +++ b/src/sites/bulk_harvest.py @@ -0,0 +1,212 @@ +"""BulkHarvestToolTimelineView's runtime-bound hammer and camera streams. + +The timeline names its clips by serialized identity. The tool and field camera +are supplied by the view at runtime, so they cannot be inferred from null scene +bindings. Keep that contract explicit, and never assign a source object identity +to the exported camera proxy. +""" +from __future__ import annotations + +import hashlib +import json +import struct +from pathlib import Path +from types import SimpleNamespace + +from core.atomic import PUBLIC_FILE_MODE, write_json +from core.gltf import GLB +from core.jsonio import normalize +from fixtures.animations import decode_clip, embed, _write_channels +from . import step_items as steps +from .scenes import PackageExtract + +PACKAGE = "mysekai__player__motion__bulk_harvest" +PREFAB = "tl_bulk_harvest_main" +VIEW = "BulkHarvestToolTimelineView" +TOOL_PACKAGE = "mysekai__tool__mdl_site_tool_hammer01" +TOOL_PREFAB = "mdl_site_tool_hammer01" +CAMERA_NODE = "moly_bulk_camera" + + +def selected_tracks(tracks, targets, director, package): + """Resolve the three explicitly runtime-bound animation streams.""" + timeline = steps._timeline(tracks, director) + selected = {} + bindings = {steps._asset(row.get("key")): row.get("value") + for row in director.get("sceneBindings", [])} + for track in steps._walk_tracks(timeline.get("tracks")): + if track.get("class") != "AnimationTrack": + continue + name = track.get("name") + if name not in ("CharacterAnimator", "Fixture1", "camera") or name in selected: + raise ValueError(f"unsupported bulk animation stream: {name!r}") + identity = steps._asset(track.get("asset")) + if steps._asset(bindings.get(identity)) is not None: + raise ValueError(f"bulk runtime stream {name!r} unexpectedly has a scene binding") + clips = [] + for row in sorted((row for row in targets.get("keyedClips", []) + if steps._asset(row.get("track")) == identity), + key=lambda row: row.get("clipIndex", 0)): + target = row.get("target") or {} + source = steps._asset(target.get("sourceClip")) + if source is None or not target.get("clipName") or not target.get("targetPackage"): + raise ValueError(f"bulk stream {name!r} has an unresolved clip") + clips.append({"name": target["clipName"], "sourceClip": { + "file": source[0], "pathId": source[1]}, + "sourcePackage": target["targetPackage"]}) + if not clips: + raise ValueError(f"bulk stream {name!r} has no clips") + selected[name] = clips + if set(selected) != {"CharacterAnimator", "Fixture1", "camera"}: + raise ValueError("bulk timeline is missing a runtime animation stream") + return selected + + +def clip_records(store, clips): + files = {} + for clip in clips: + package = store.package(clip["sourcePackage"]) + if package is None: + raise ValueError(f"missing clip package: {clip['sourcePackage']}") + records = steps._Records(store, package) + found = records.lookup(steps._asset(clip["sourceClip"])) + if found is None or found[0].kinds[found[1]] != "AnimationClip": + raise ValueError(f"clip source is not an AnimationClip: {clip['sourceClip']}") + record, pid = found + if str(record.tree(pid).get("m_Name")) != clip["name"]: + raise ValueError(f"clip name disagrees with source: {clip['sourceClip']}") + files[record.archive] = record + return SimpleNamespace(lookup=lambda identity: ( + (files[identity[0]], int(identity[1])) + if identity[0] in files and int(identity[1]) in files[identity[0]].kinds else None)) + + +def _load_glb(path): + data = path.read_bytes() + if data[:4] != b"glTF" or struct.unpack_from(" len(indices): + raise ValueError("triangle submesh exceeds the index buffer") + # Unity's indices are relative to baseVertex. MeshHandler.get_triangles + # does not apply that offset, so read the authored triangle ranges here. + base = getattr(submesh, "baseVertex", None) or 0 + for offset in range(start, start + count, 3): + triangle = [int(index) + base for index in indices[offset:offset + 3]] + if any(index < 0 or index >= len(positions) for index in triangle): + raise ValueError("triangle index outside the vertex buffer") + triangles.append(triangle) + if not triangles: + raise ValueError("collider mesh has no triangles") + return positions, triangles + + +def _colliders(package, tag_index): + colliders = [] + for record in sorted(package.files, key=lambda item: item.archive): + graph = None + for path_id, kind in sorted(record.kinds.items()): + if not (kind.endswith("Collider") or kind == "CharacterController"): + continue + collider = record.tree(path_id) + owner = _local_target(record, collider.get("m_GameObject"), "GameObject") + game_object = record.tree(owner) + tag = game_object.get("m_Tag") + if isinstance(tag, bool) or not isinstance(tag, int) or tag < 0: + raise ValueError("missing or invalid GameObject tag") + if tag != tag_index: + continue + if kind != "MeshCollider": + raise ValueError(f"unsupported {TAG} component: {kind}") + mesh_id = _local_target(record, collider.get("m_Mesh"), "Mesh") + if graph is None: + graph = Graph(record) + positions, triangles = _mesh_geometry(record.objects[mesh_id].read()) + layer = game_object.get("m_Layer") + if isinstance(layer, bool) or not isinstance(layer, int) or not 0 <= layer < 32: + raise ValueError("missing or invalid GameObject layer") + colliders.append({ + "tag": TAG, "tagIndex": tag_index, + "enabled": _flag(collider, "m_Enabled"), + "isTrigger": _flag(collider, "m_IsTrigger"), + "convex": _flag(collider, "m_Convex"), "layer": layer, + "chain": _chain(record, graph, owner), + "positions": positions, "triangles": triangles, + "source": {"file": record.archive, "componentId": str(path_id), + "gameObjectId": str(owner), "meshId": str(mesh_id)}, + }) + if not colliders: + raise ValueError(f"scene has no {TAG} colliders") + return colliders + + +def extract_site_edit_pick(bundle, apks, out): + """Export one scene sidecar from its source bundle and matching client APKs. + + All pointers used by schema v1 must stay in the collider's serialized file; + otherwise its single-file provenance would be incomplete, so fail rather + than publish a partial surface. Other scene dependencies are not rendered + or loaded. The explicit output file is replaced atomically after success. + """ + configure_fallback_unity_version() + bundle, apks, out = Path(bundle), [Path(apk) for apk in apks], Path(out) + require_paths([bundle]) + name = bundle.name + site = name.removeprefix(SCENE_PREFIX) + if not name.startswith(SCENE_PREFIX) or not site: + raise ValueError(f"bundle must be named {SCENE_PREFIX}") + if out.resolve() in {path.resolve() for path in [bundle, *apks]}: + raise ValueError("output must not replace an input") + tag_index, tag_source = _tag_manager(apks) + store = PackageStore([bundle]) + package = store.package(name) + if package is None: + raise ValueError(f"scene package not loaded: {name}") + document = { + "version": 1, "coordinateContract": COORDINATE_CONTRACT, + "site": site, "package": name, + "bundleSha256": store.source_receipt(name)["sha256"], + "colliders": _colliders(package, tag_index), "tagManagerSource": tag_source, + } + payload = json.dumps(document, ensure_ascii=False, separators=(",", ":"), + allow_nan=False).encode("utf-8") + out.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=out.parent, prefix=f".{out.name}.", + delete=False) as handle: + temporary = Path(handle.name) + handle.write(payload) + os.replace(temporary, out) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return {"output": str(out), "site": site, "colliders": len(document["colliders"]), + "vertices": sum(len(row["positions"]) for row in document["colliders"]), + "triangles": sum(len(row["triangles"]) for row in document["colliders"])} diff --git a/src/sites/step_items.py b/src/sites/step_items.py index 7f01b2d..48135bc 100644 --- a/src/sites/step_items.py +++ b/src/sites/step_items.py @@ -393,7 +393,8 @@ def export_site_timelines(bundles_root, asset_root, packages): paths = [bundles / name for name in packages] require_paths([str(path) for path in paths]) for name in packages: - if not name.startswith(SITE_PREFIX): + from .bulk_harvest import PACKAGE as BULK_PACKAGE + if not name.startswith(SITE_PREFIX) and name != BULK_PACKAGE: raise ValueError(f"{name} is not a site package") output = root / OUTPUT staged = output.with_name(output.name + ".staging") @@ -416,7 +417,11 @@ def export_site_timelines(bundles_root, asset_root, packages): targets = json.loads(targets_path.read_text(encoding="utf-8")) package = store.package(name) records = _Records(store, package) - items, item_reports = export_step_items(store, records, name, tracks, targets, step_dir) + if name == BULK_PACKAGE: + from .bulk_harvest import export_bulk_harvest + items, item_reports = export_bulk_harvest(store, records, tracks, targets, step_dir) + else: + items, item_reports = export_step_items(store, records, name, tracks, targets, step_dir) containers = {row["container"] for row in items.values()} directors, director_reports = export_scene_directors(store, records, name, tracks, targets, step_dir, containers) diff --git a/src/source/__init__.py b/src/source/__init__.py new file mode 100644 index 0000000..b75d8ec --- /dev/null +++ b/src/source/__init__.py @@ -0,0 +1 @@ +"""Caller-supplied, content-verified sources used by extraction tasks.""" diff --git a/src/source/snapshot.py b/src/source/snapshot.py new file mode 100644 index 0000000..d94e60c --- /dev/null +++ b/src/source/snapshot.py @@ -0,0 +1,47 @@ +"""Portable source receipts: content identity is independent of its locator. + +Receipts and plans can contain local paths. They are private build inputs, +not runtime artifacts. A task verifies the bytes it actually reads instead +of trusting file times or a previously computed hash. +""" +from pathlib import Path + +from core.content import content_hash, document_hash + + +def file_input(role, path): + path = Path(path).resolve(strict=True) + data = path.read_bytes() + return {"role": role, "sha256": content_hash(data), "size": len(data), + "locators": [{"kind": "file", "path": str(path)}]} + + +def read_inputs(inputs): + frozen = {} + for item in inputs: + failures = [] + for locator in item["locators"]: + try: + if locator.get("kind") != "file": + raise ValueError("unsupported input locator") + data = Path(locator["path"]).read_bytes() + if len(data) != item["size"] or content_hash(data) != item["sha256"]: + raise ValueError("input content changed after planning") + frozen[item["role"]] = data + break + except (OSError, ValueError) as exc: + failures.append(str(exc)) + else: + raise ValueError(f"cannot read verified input {item['role']}: {'; '.join(failures)}") + return frozen + + +def snapshot(inputs, *, region, version): + if region not in {"jp", "cn", "tw", "en", "kr"} or not version: + raise ValueError("source requires a known region and an explicit version") + inputs = sorted(inputs, key=lambda item: item["role"]) + if len({item["role"] for item in inputs}) != len(inputs): + raise ValueError("duplicate source input role") + identity = {"schema": "moly-source/1", "region": region, "version": version, + "inputs": [{k: item[k] for k in ("role", "sha256", "size")} for item in inputs]} + return {**identity, "revision": document_hash(identity), "inputs": inputs} diff --git a/src/ui/action_icon.py b/src/ui/action_icon.py index ab615d1..782cc7e 100644 --- a/src/ui/action_icon.py +++ b/src/ui/action_icon.py @@ -26,7 +26,7 @@ import UnityPy -from core.extract import DEFAULT_UNITY_VERSION +from core.unity import configure_fallback_unity_version from core.jsonio import write_json # Identity of the bundle root this toolchain is pinned to. Both go into the @@ -117,7 +117,7 @@ def export_action_icons( "version": 1, "region": region, "gameVersion": game_version, - "unityVersion": DEFAULT_UNITY_VERSION, + "unityVersion": configure_fallback_unity_version(), "summary": {"bundles": len(paths), "icons": len(records)}, "icons": records, } diff --git a/src/ui/editor_layout.py b/src/ui/editor_layout.py index a940ac2..fe4bd30 100644 --- a/src/ui/editor_layout.py +++ b/src/ui/editor_layout.py @@ -8,7 +8,7 @@ "ScreenLayerSiteEditMode", "MysekaiEditSaveConfirmationDialog", "SiteEditListSelectorCell", "SiteEditOutDoorContentList", "SiteEditFloorContentList", "SiteEditWallContentList", "SiteEnvironmentContentList", "RoadFixtureContentList", - "FixtureSelectCell", "UIPartsLeftTabListMysekaiContentCell", + "FixtureSelectCell", "SiteEnvironmentSelectCell", "UIPartsLeftTabListMysekaiContentCell", "ScreenLayerHeader", ) @@ -177,6 +177,18 @@ def decode_left_tab_controller(r): return d +def decode_site_environment_cell(r): + """ListViewItem fields, then SiteEnvironmentSelectCell's two references.""" + return {"sizeX": r.f32(), "sizeY": r.f32(), + **refs(r, ("_thumbnail", "_selected"))} + + +def decode_site_environment_thumbnail(r): + """UIPartsThumbnail fields, then its environment button and detector.""" + from ui.bgm_select_layout import decode_thumbnail + return {**decode_thumbnail(r), **refs(r, ("button", "clickDetector"))} + + def decode_fixture_cell(r): d = {"sizeX": r.f32(), "sizeY": r.f32()} d.update(refs(r, ("thumbnail", "selected", "_inPlacedLabel", "_missionLabel"))) @@ -301,6 +313,8 @@ def decode_layer_data(r): "Sekai.UI.UITextureLoader": decode_texture_loader, "Sekai.UI.UIPartsLeftTabListController": decode_left_tab_controller, "Sekai.UI.FixtureSelectCell": decode_fixture_cell, + "Sekai.UI.SiteEnvironmentSelectCell": decode_site_environment_cell, + "Sekai.Mysekai.UIPartsSiteEnvironmentThumbnail": decode_site_environment_thumbnail, "Sekai.UI.UIPartsIconInfoView": lambda r: refs(r, ("infoTextRoot", "infoText", "lockedObject")), "Sekai.UI.UIPartsLeftTabList": lambda r: refs(r, ("scroller", "topScrollableIcon", "bottomScrollableIcon", "backPanelTransform")), "Sekai.UI.UIPartsLeftTabSubList": lambda r: refs(r, ("scroller", "topScrollableIcon", "bottomScrollableIcon")), diff --git a/src/ui/jp_decoders.py b/src/ui/jp_decoders.py index b36216f..cbaea38 100644 --- a/src/ui/jp_decoders.py +++ b/src/ui/jp_decoders.py @@ -141,6 +141,7 @@ from ui import bgm_select_layout from ui import mission_layout from ui import convert_layout +from ui import photo_layout REGION = "jp" @@ -533,7 +534,83 @@ def decode_list_view_700(r: talk.Reader) -> dict: DECODERS_681 = {**SHARED, **ONLY_681} # The 7.0.0 layout: the four dialogs end at their click event, and the five # classes the 7.0.0 declaration changed read their added fields. + + +def decode_bulk_checkbox_control(r): + return {"checkObj": r.pptr(), "check": r.bool4(), "toggle": r.pptr()} + + +def decode_harvest_player_hud_700(r): + return editor_layout.refs(r, ("_staminaView", "_bulkHarvestChargeGaugeView", "_canvasGroup")) + + +def decode_stock_scroll_rect(r): + fields = {"m_Content": r.pptr(), "m_Horizontal": r.bool4(), "m_Vertical": r.bool4(), + "m_MovementType": r.i32(), "m_Elasticity": r.f32(), "m_Inertia": r.bool4(), + "m_DecelerationRate": r.f32(), "m_ScrollSensitivity": r.f32()} + fields.update(editor_layout.refs(r, ("m_Viewport", "m_HorizontalScrollbar", "m_VerticalScrollbar"))) + fields.update(m_HorizontalScrollbarVisibility=r.i32(), m_VerticalScrollbarVisibility=r.i32(), + m_HorizontalScrollbarSpacing=r.f32(), m_VerticalScrollbarSpacing=r.f32(), + m_OnValueChanged=talk.decode_unity_event(r)) + return fields + +def decode_bulk_harvest_button(r): + fields = decode_custom_button(r) + fields.update(editor_layout.refs(r, ("_cannotButton", "_cannotBalloon", "_iconLoader"))) + return fields + + +def decode_bulk_harvest_gauge(r): + return editor_layout.refs(r, ("_gaugeImage", "_canvasGroup", "_maxEffectRoot")) + + +def decode_bulk_harvest_settings(r): + from ui.window_animation import decode_common2_button_dialog + fields = decode_common2_button_dialog(r) + fields.update(editor_layout.refs(r, ("_siteTabGroup", "_checkBoxListViewPrefab", "_checkBoxListViewParent"))) + return fields + + +def decode_bulk_harvest_list(r): + return editor_layout.refs(r, ("_allCheckBox", "_groupViewRoot", "_groupViewBase")) + + +def decode_bulk_harvest_group(r): + return editor_layout.refs(r, ("_groupNameText", "_allCheckBox", "_checkBoxRoot", "_checkBoxBase")) + + +def decode_bulk_harvest_checkbox(r): + return editor_layout.refs(r, ("_checkBox", "_text")) + + +def decode_bulk_harvest_reward(r): + from ui.window_animation import decode_common1_button_dialog + fields = decode_common1_button_dialog(r) + fields.update(editor_layout.refs(r, ("_listView", "_shortageTextRoot", "_staminaShortageText", "_toolShortageText"))) + return fields + + +def decode_harvest_layer_700(r): + fields = {"layerCamera": r.pptr()} + fields.update(editor_layout.refs(r, ("_menuUIContent", "_selectedToolView", "_toolSelectorViewController", + "_harvestActionButton", "_toolNotHaveBalloon", "_harvestActionCannotButton", "_dashButton", "_bulkHarvestActionButton"))) + return fields + ONLY_700 = { + "Sekai.UIPartsThumbnailRewardListCell": lambda r: {"sizeX": r.f32(), "sizeY": r.f32(), "thumbnailRoots": talk.decode_pptr_list(r), "nameText": r.pptr()}, + "Sekai.UIPartsCheckBox": decode_bulk_checkbox_control, + "Sekai.Mysekai.HarvestPlayerHeadUpDisplay": decode_harvest_player_hud_700, + "UnityEngine.UI.ScrollRect": decode_stock_scroll_rect, + "Sekai.Mysekai.BulkHarvestActionButton": decode_bulk_harvest_button, + "Sekai.Mysekai.BulkHarvestChargeGaugeView": decode_bulk_harvest_gauge, + "Sekai.Mysekai.BulkHarvestCheckBoxListView": decode_bulk_harvest_list, + "Sekai.Mysekai.BulkHarvestGroupCheckBoxView": decode_bulk_harvest_group, + "Sekai.Mysekai.BulkHarvestTargetSettingCheckBox": decode_bulk_harvest_checkbox, + "Sekai.Mysekai.BulkHarvestTargetSettingDialog": decode_bulk_harvest_settings, + "Sekai.Mysekai.MysekaiBulkHarvestRewardDialog": decode_bulk_harvest_reward, + "Sekai.Mysekai.ScreenLayerMysekaiHarvest": decode_harvest_layer_700, + + **photo_layout.DECODERS, "Sekai.UI.CustomScrollRect": decode_custom_scroll_rect_700, "Sekai.Mysekai.MysekaiMissionStartDialog": mission_layout.decode_mission_start_dialog, "Sekai.Mysekai.MysekaiMissionCompleteDialog": mission_layout.decode_mission_complete_dialog, @@ -560,6 +637,28 @@ def decode_list_view_700(r: talk.Reader) -> dict: # (class name, serialized type hash hex) -> the layout the bytes carry. TYPE_LAYOUTS = { + ("Sekai.UIPartsThumbnailRewardListCell", "d2817fbfe87a2ef02b700815f2918477"): "7.0.0", + ("Sekai.Mysekai.HarvestPlayerHeadUpDisplay", "b0aefa4373b8001001f4e51227165a4e"): "7.0.0", + ("Sekai.UIPartsCheckBox", "beb6fd08a47ed26ebf63cc74f518fb08"): "7.0.0", + ("UnityEngine.UI.ScrollRect", "d5cd7f207e91ca8336c65a577a6fb2a0"): "7.0.0", + + ("Sekai.Mysekai.BulkHarvestActionButton", "0502d10b158ddd2b9dd787bb9dd58fb4"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestChargeGaugeView", "2167f64c1ddff3c0aece99da045ee93c"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestCheckBoxListView", "d2835e1e2684fe21d4e3bddcd2841cf3"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestGroupCheckBoxView", "b3511010ceb423f8f82c90a70d17ce65"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestTargetSettingCheckBox", "5ad4d5bf1f146309431caa4cfd96f525"): "7.0.0", + ("Sekai.Mysekai.BulkHarvestTargetSettingDialog", "1faddd3b4959d3e99ca1afa0d2c7ac43"): "7.0.0", + ("Sekai.Mysekai.MysekaiBulkHarvestRewardDialog", "7231bc0e697f6ce50d2b7ce254736217"): "7.0.0", + ("Sekai.Mysekai.ScreenLayerMysekaiHarvest", "4dda12598a43e512cb119ab96403b0a9"): "7.0.0", + + ("Sekai.UIPartsPhotoFrameThumbnail", "701a1b9241173c653af93358697fbf4a"): "7.0.0", + ("Sekai.Mysekai.MysekaiSelectFrameView", "56a5828cb66f421adde139868f381dc6"): "7.0.0", + ("Sekai.UIPartsPhotoFramePreview", "8d1a4f81285d407da7fc2608b914a06e"): "7.0.0", + ("Sekai.Mysekai.MysekaiPhotoSaveAnimation", "a7952c6374ff1805f356704076fb3a8a"): "7.0.0", + ("Sekai.AnimationClipEvent", "431f36454113f7d83bc74347ac5d008f"): "7.0.0", + ("Sekai.Mysekai.MysekaiPhotoShotView", "9c249c4e2d9276be8c161151ab811ebf"): "7.0.0", + ("Sekai.Mysekai.MysekaiPhotoShotAnimation", "f1b1ff8667c818bf70d56dd9b284b432"): "7.0.0", + ("Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter", "a4805d90303631a00c5a8e37e1ff5a00"): "7.0.0", ("Sekai.ColorFader", "2a6b976ee5ac7d8120ec45875b46424f"): "6.8.1", ("Sekai.CrystalShop.ShopConfirmDialogPriceParts", "2a424898f46b3aeb4201e3430b854139"): "6.8.1", ("Sekai.EventBreakTimeController", "273defc7d306842f06a7362af5082289"): "6.8.1", diff --git a/src/ui/photo_layout.py b/src/ui/photo_layout.py new file mode 100644 index 0000000..8eaf8a7 --- /dev/null +++ b/src/ui/photo_layout.py @@ -0,0 +1,198 @@ +"""Serialized photo view references, separate from runtime photo state.""" +from ui import editor_layout, talk +from pathlib import Path +import hashlib +import UnityPy + +PACKAGE = "mysekai__ui__mysekai_photo_shot" +IMAGES = ( + "overlay_photoshot_screen", "frame_PhotoShot", "line_cornerguide_photoshot", + "btn_photoshot_gr_h180", "icon_callout_wh", "icon_camera_wh_h80", "btn_circle120_lgr", + "icon_cameraEffect_h56", "icon_tripod_wh_h44", "icon_photoalbum_wh_h42", + "txt_photoshot_blur", "txt_photoshot_filmCount", + "tex_mysekai_photo_shot_camera", "tex_mysekai_photo_shot_parts", "img_film_default", +) + + +def package_images(data, out): + """The view loads images by these authored names when it opens.""" + env = UnityPy.load(data) + sprites = {} + for obj in env.objects: + if obj.type.name != 'Sprite': + continue + tree = obj.read_typetree() + # A bundle also includes common-atlas sprite aliases whose atlas data + # is supplied elsewhere. The photo view's own standalone sprite is + # the one with a concrete unpacked texture and no atlas binding. + if tree.get('m_SpriteAtlas', {}).get('m_PathID') or tree['m_RD']['settingsRaw'] & 1: + continue + name = tree['m_Name'] + if name in sprites: + raise ValueError(f'duplicate unpacked photo sprite {name}') + sprites[name] = obj + textures = {obj.read_typetree()["m_Name"]: obj for obj in env.objects if obj.type.name == "Texture2D"} + paths, metadata = {}, {} + output = Path(out) + (output / 'textures').mkdir(parents=True, exist_ok=True) + for name in IMAGES: + sprite = sprites.get(name) + obj = sprite or textures.get(name) + if obj is None: + raise ValueError(f'photo package is missing authored image {name}') + relative = f'textures/photo-{name}.png' + image = obj.read() + if sprite is not None and not image.m_SpriteAtlas: + # These unpacked sprites retain a CommonAtlas tag even though the + # package's atlas is empty. Their own render-data texture is the + # authored source; do not resolve a tag to an unrelated atlas. + if image.m_RD.settingsRaw & 1 or not image.m_RD.texture: + raise ValueError(f'photo sprite has no unpacked texture: {name}') + image.m_AtlasTags = [] + image.image.save(output / relative) + paths[name] = relative + if sprite is not None: + tree = sprite.read_typetree() + rect, rd = tree['m_Rect'], tree['m_RD'] + metadata[name] = { + 'name': name, 'image': relative, 'package': PACKAGE, 'pathId': obj.path_id, + 'rect': [rect[k] for k in ('x', 'y', 'width', 'height')], + 'rectSize': [rect['width'], rect['height']], + 'border': [tree['m_Border'][k] for k in ('x', 'y', 'z', 'w')], + 'pixelsPerUnit': tree['m_PixelsToUnits'], + 'pivot': [tree['m_Pivot']['x'], tree['m_Pivot']['y']], + 'textureRect': [rd['textureRect'][k] for k in ('x', 'y', 'width', 'height')], + 'textureRectOffset': [rd['textureRectOffset']['x'], rd['textureRectOffset']['y']], + 'packingSettingsRaw': rd['settingsRaw'], + 'downscaleMultiplier': rd.get('downscaleMultiplier', 1.0), + } + return paths, metadata, {'package': PACKAGE, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), + 'images': len(paths), 'sprites': len(metadata)} + + +def package_effects(bundle, bundle_root, out): + """Export each dynamically loaded effect with only its reachable objects. + + These prefabs share node names inside one package. Exporting all roots + together makes a UI particle's node binding ambiguous and includes unrelated + atlas aliases. The existing prefab view follows the selected root's actual + component, material and texture references before the normal scene export. + """ + from core.assets.packages import PackageStore + from core.jsonio import write_json + from core.player_resources import prefab_view + from sites.scenes import PackageExtract + + store = PackageStore([bundle], bundle_root) + store.load_dependencies([Path(bundle).name]) + package = store.package(Path(bundle).name) + records = [] + for role, prefab in ( + ('call-button', 'fx_mysekai_photo_shot_call_character_pt'), + ('call-wave', 'fx_mysekai_photo_shot_call_character_wave'), + ('shot', 'fx_mysekai_photo_shot_pt'), + ): + targets = [(record, path_id) for name, record, path_id in package.contents + if name == prefab + '.prefab'] + if len(targets) != 1: + raise ValueError(f'photo effect {prefab} needs exactly one prefab root') + view, report = prefab_view(store, targets, f'photo-shot:{role}') + if report['unreadable'] or report.get('unfollowedArchives'): + raise ValueError(f'photo effect {prefab} has an incomplete object closure: {report}') + store.adopt(view) + directory = Path(out) / 'photo-shot' / role + document = PackageExtract(store, view.name, directory, role, + classification=('prefab-package', role)).run() + primary = [root for root in document['roots'] if root['primary']] + if len(primary) != 1 or primary[0]['name'] != prefab or document['unsupported']: + raise ValueError(f'photo effect {prefab} is incomplete: {document["unsupported"]}') + document['source'] = {'package': store.source_receipt(Path(bundle).name), + 'prefab': prefab, 'view': report} + relative = f'photo-shot/{role}/{role}.json' + write_json(Path(out) / relative, document) + records.append({'document': relative, 'kind': 'photo-particle', + 'documentSha256': hashlib.sha256((Path(out) / relative).read_bytes()).hexdigest(), + 'source': document['source'], 'particles': len(document['particles'])}) + return records + + +VIEW_FIELDS = ( + "_canvasGroup", "_overlayImage", "_frameImage", "_lineCornerImageList", + "_playerHeadUpDisplay", "_switchEditModeToggle", "_avatarMotionSelector", + "_photoShotBaseImage", "_photoShotIconImage", "_photoShotButton", + "_photoShotCoverBaseImage", "_photoShotIconCoverImage", "_photoShotCoverButton", + "_callCharacterButton", "_callCharacterButtonCover", "_callCharacterButtonIcon", + "_callCharacterButtonCoverIcon", "_callCharacterButtonGraphics", "_callCharacterButtonTapEffect", + "_callCharacterWaveParticleRoot", "_iconBaseImageList", "_photoEffectIconImage", + "_openAvatarMotionSelectorButton", "_showsSelectPhotoEffectDialogButton", "_openPhotoAlbumButton", + "_dofAdjustSlider", "_tripodIconList", "_photoAlbumIcon", "_remainFilmCount", + "_remainFilmCountBg", "_dofTextImage", "_remainCountTextImage", "_callCharacterTutorialGuidePrefab", +) +VIEW_ARRAYS = {"_lineCornerImageList", "_callCharacterButtonGraphics", "_iconBaseImageList", "_tripodIconList"} +ANIMATION_FIELDS = ( + "_fxPhotoShotRoot", "_photoShotAnimator", "_photoFramePreview", "_photoCameraImages", + "_photoFramePartsImage", "_photoFilmImage", "_photoFrameShadowCanvasGroup", "_showPhotoFramePreviewEvent", +) + + +def references(reader, fields, arrays=()): + result = {} + for name in fields: + if name not in arrays: + result[name] = reader.pptr() + continue + count = reader.i32() + if count < 0 or count > reader.remaining() // 12: + raise ValueError('photo reference array over-reads object') + result[name] = [reader.pptr() for _ in range(count)] + return result + + +def decode_view(reader: talk.Reader): + return references(reader, VIEW_FIELDS, VIEW_ARRAYS) + + +def decode_animation(reader: talk.Reader): + return references(reader, ANIMATION_FIELDS, {"_photoCameraImages"}) + + +def decode_presenter(reader: talk.Reader): + return editor_layout.refs(reader, ("_photoShotView", "_selectFrameView", "_photoSaveAnimation")) + + +def decode_select_frame(reader: talk.Reader): + return references(reader, ( + "_canvasGroup", "_photoShotAnimation", "_scroller", + "_selectedPhotoFrameNameText", "_savePhotoButton", "_photoShotSkipButton", + )) + + +def decode_preview(reader: talk.Reader): + return references(reader, ( + "_rootCanvasGroup", "_photoLoader", "_frameLoader", "_frameCanvasGroup", + )) + + +def decode_save_animation(reader: talk.Reader): + return references(reader, ("_canvasGroup", "_photoAlbumPageView")) + + +def decode_thumbnail(reader: talk.Reader): + return references(reader, ("_frameLoader", "_quantityText", "_selectedImage", "_selectedButton")) + + +def decode_clip_event(reader: talk.Reader): + # Callback is a runtime Action, not a serialized UnityEvent. + return {} + + +DECODERS = { + "Sekai.Mysekai.MysekaiPhotoShotView": decode_view, + "Sekai.Mysekai.MysekaiPhotoShotAnimation": decode_animation, + "Sekai.Mysekai.ScreenLayerMysekaiPhotoShotPresenter": decode_presenter, + "Sekai.Mysekai.MysekaiSelectFrameView": decode_select_frame, + "Sekai.UIPartsPhotoFramePreview": decode_preview, + "Sekai.Mysekai.MysekaiPhotoSaveAnimation": decode_save_animation, + "Sekai.AnimationClipEvent": decode_clip_event, + "Sekai.UIPartsPhotoFrameThumbnail": decode_thumbnail, +} diff --git a/src/ui/photo_presentation.py b/src/ui/photo_presentation.py new file mode 100644 index 0000000..7e1cc4a --- /dev/null +++ b/src/ui/photo_presentation.py @@ -0,0 +1,98 @@ +"""Playable PhotoShot curves and the master's frame image inventory. + +The inventory is not a possession list. Possession is a separate client input. +Curve targets are resolved from the animator's actual bindings to the exported +screen identities; no hierarchy-name lookup is needed by the runtime player. +""" +from pathlib import Path +import hashlib +import json + +import UnityPy +from core.jsonio import write_json + + +def export_presentation(document, master_dir, bundle_root, out): + nodes = document['nodes'] + animation = next(c for n in nodes for c in n['components'] + if c['class'] == 'Sekai.Mysekai.MysekaiPhotoShotAnimation') + animator_id = animation['fields']['_photoShotAnimator'][1] + animator = next(c for n in nodes for c in n['components'] if c['pathId'] == animator_id) + source = animator['animatorController'] + state = next(s for m in source['controller']['stateMachines'] for s in m['states'] + if s['name'] == 'PhotoShot') + if len(state['motions']) != 1 or state['loop'] or state['speed'] != 1: + raise ValueError('PhotoShot requires one non-looping motion at authored speed') + motion = state['motions'][0] + clip = next(c for c in source['clips'] if c['source'] == motion['clip']['source']) + if clip['accounting']['unresolved'] or clip['accounting']['bindingSlots'] != len(clip['curves']): + raise ValueError('PhotoShot has unresolved animation slots') + properties = { + (224, 1460864421): ('position', 0), (224, 538195251): ('position', 1), + (114, 2526845255): ('color', 0), (114, 4215373228): ('color', 1), + (114, 2334886179): ('color', 2), (114, 304273561): ('color', 3), + (225, 1574349066): ('alpha', 0), (1, 2086281974): ('active', 0), + } + targets = {} + curves = [] + for curve in clip['curves']: + if len(curve['targets']) != 1 or curve['kind'] not in ('cubic', 'const', 'linear'): + raise ValueError('PhotoShot curve needs one exact target and supported interpolation') + target, binding = curve['targets'][0], curve['binding'] + game_object_id = int(target['gameObject']['pathId']) + node = next(n for n in nodes if n['gameObjectId'] == game_object_id) + if binding['typeId'] == 4 and binding['attribute'] in (3, 4): + prop, channel = ('scale' if binding['attribute'] == 3 else 'euler'), binding['component'] + else: + prop, channel = properties[(binding['typeId'], binding['attribute'])] + component_id = None + if prop == 'color': + if len(target['components']) != 1: + raise ValueError('PhotoShot color needs its exact Graphic component') + component_id = int(target['components'][0]['pathId']) + key = node['transformId'] + if key not in targets: + graphic = next((c for c in node['components'] if c['pathId'] == component_id), None) + targets[key] = {'transformId': node['transformId'], 'graphicId': component_id, + 'selectorId': next((c['pathId'] for c in node['components'] + if c.get('fields', {}).get('m_Color') is not None), + node['transformId']), + 'position': node['rect']['anchoredPosition'], 'scale': node['rect']['localScale'], + 'color': graphic['fields']['m_Color'] if graphic else [1., 1., 1., 1.]} + if component_id is not None: + graphic = next(c for c in node['components'] if c['pathId'] == component_id) + targets[key]['graphicId'] = component_id + targets[key]['color'] = graphic['fields']['m_Color'] + curves.append({'target': list(targets).index(key), 'property': prop, 'channel': channel, + 'kind': curve['kind'], 'points': curve['points'], 'sourceSlot': curve['slot']}) + frames = [] + output = Path(out) + for row in json.loads((Path(master_dir) / 'mysekaiPhotoDecorations.json').read_text(encoding='utf-8')): + frame = {'id': row['id'], 'name': row['name'], 'assetbundleName': row['assetbundleName']} + for role in ('preview', 'thumbnail'): + package = f"mysekai__photo_frame__{role}__{row['assetbundleName']}" + path = Path(bundle_root) / package + data = path.read_bytes() + env = UnityPy.load(data) + matches = [o for o in env.objects if o.type.name == 'Texture2D' + and o.read_typetree()['m_Name'] == row['assetbundleName']] + if len(matches) != 1: + raise ValueError(f'{package} needs exactly one named frame texture') + texture = matches[0] + relative = f"photo-shot/frames/{row['id']}-{role}.png" + (output / relative).parent.mkdir(parents=True, exist_ok=True) + texture.read().image.save(output / relative) + frame[role] = relative + frame[f'{role}Source'] = {'package': package, 'pathId': texture.path_id, + 'sha256': hashlib.sha256(data).hexdigest()} + frames.append(frame) + result = {'version': 1, 'duration': clip['duration'], 'targets': list(targets.values()), + 'curves': curves, 'events': clip['events'], 'frames': frames, + 'source': {'controller': source['controller']['source'], 'clip': clip['source'], + 'animatorId': animator_id, 'clipIndex': motion['clipIndex'], + 'accounting': clip['accounting']}} + relative = 'photo-shot/presentation.json' + path = write_json(output / relative, result) + return {'document': relative, 'kind': 'photo-presentation', + 'documentSha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'curves': len(curves), 'frames': len(frames)} diff --git a/src/ui/prefab_layout.py b/src/ui/prefab_layout.py index 1b3eccb..4b6288f 100644 --- a/src/ui/prefab_layout.py +++ b/src/ui/prefab_layout.py @@ -142,6 +142,7 @@ # The mission cells sit in Resources folders and are instantiated from # their lists' serialized cell prefab reference (see POINTER_PREFABS). "missioncell": ("MysekaiMissionScreenCell", "MysekaiMissionRewardCell"), + "photocell": ("UIPartsPhotoFrameListThumbnail",), } # Screen layers and dialogs the JP UI census maps to but the product does not @@ -275,6 +276,7 @@ # The JP prefab registry: the product families, the referenced-only prefabs # and the census-only ones. JP_FAMILIES = { + "bulkharvest": ("BulkHarvestChargeGauge", "UIPartsThumbnailRewardListCell"), **SCREENS, **REFERENCED_FAMILIES, **{family: tuple(n for n in names if n not in _PRODUCT_PREFABS) @@ -288,7 +290,15 @@ # repeats its page, whose pages each hold a copy of the content beside the list's # cell instance, adds a fourth item, the node's path under the page, and builds # the prefab from the page's own nodes (see follow_page_prefab_reference). -POINTER_PREFABS = {**CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS} +POINTER_PREFABS = { + "BulkHarvestChargeGauge": ("ScreenLayerMysekaiHUD", "Sekai.Mysekai.HarvestPlayerHeadUpDisplay", "_bulkHarvestChargeGaugeView"), + "UIPartsThumbnailRewardListCell": ("MysekaiBulkHarvestRewardDialog", "Sekai.UI.ListView", "listViewItemPrefab"), + "SiteEnvironmentSelectCell": ( + "SiteEnvironmentContentList", "Sekai.UI.ListView", "listViewItemPrefab"), + **CRAFT_POINTER_PREFABS, **MISSION_POINTER_PREFABS, + "UIPartsPhotoFrameListThumbnail": ( + "ScreenLayerMysekaiPhotoShot", "Sekai.FixedSizeReuseScroller", "prefab"), +} POINTER_PAGE_PREFABS = {**MISSION_PAGE_POINTER_PREFABS} # Prefabs -> (holder, the holder's list field, the cell's name): the field names @@ -794,6 +804,7 @@ def decode_screen_layer_data_list(r: talk.Reader) -> dict: **FIELD_HUD_CN_DECODERS, "UnityEngine.UI.Toggle": decode_unity_toggle, "Sekai.UI.CustomRawImage": decode_raw_image_base, + "UnityEngine.UI.RawImage": decode_raw_image_base, "Sekai.UI.CustomToggle": decode_custom_toggle, "Sekai.UI.CustomIndexToggleGroup": decode_custom_index_toggle_group, "Sekai.MenuDialogCell": decode_menu_dialog_cell, @@ -3050,6 +3061,14 @@ def extract_layout_jp(apks, out_dir, *, client_version, master_dir, prefabs, sitemap_package = Path(bundles_root) / SITEMAP_TEXTURE_PACKAGE if not sitemap_package.is_file(): raise FileNotFoundError(f"UI texture package missing: {sitemap_package.name}") + from ui import photo_layout + photo_package = None + if "ScreenLayerMysekaiPhotoShot" in selected: + if bundles_root is None: + raise ValueError("photo-shot UI needs its decrypted texture package") + photo_package = Path(bundles_root) / photo_layout.PACKAGE + if not photo_package.is_file(): + raise FileNotFoundError(f"UI texture package missing: {photo_package.name}") bgm_player_package = None if "ScreenLayerMysekaiBGMSelect" in selected: if bundles_root is None: @@ -3211,7 +3230,7 @@ def resource_key(prefab): reference, (go_pid, transform_pid, _) = follow_prefab_reference( env, holder_objects, holder_roots[0][1], item["via"][1], item["via"][2], Resolver({}, mono_index, None, env=env, decoders=decoders), - lambda name: _file_objects(env, name)) + lambda name: _file_objects(env, name), allow_nested=item["prefab"] == "BulkHarvestChargeGauge") if "targetSerializedFile" not in reference: reference["targetSerializedFile"] = holder_file item["target"] = (reference["targetSerializedFile"], go_pid) @@ -3362,6 +3381,16 @@ def digest(path): runtime_textures.update(textures) runtime_sprites.update(sprites) texture_sources.append(source) + if photo_package is not None: + textures, sprites, source = photo_layout.package_images(photo_package.read_bytes(), out) + for alias in textures: + if alias in runtime_textures: + raise ValueError(f"UI runtime texture alias collision: {alias}") + runtime_textures.update(textures) + runtime_sprites.update(sprites) + texture_sources.append(source) + for effect in photo_layout.package_effects(photo_package, bundles_root, out): + rows.append({**effect, **identity}) for item in plan: prefab, family = item["prefab"], item["family"] file_name, root_pid = item["target"] @@ -3472,6 +3501,10 @@ def digest(path): runtime_textures[alias] = sprite["image"] runtime_sprites[alias] = sprite path = write_json(out / family / f"{prefab}.json", document) + if prefab == "ScreenLayerMysekaiPhotoShot": + from ui.photo_presentation import export_presentation + presentation = export_presentation(document, master_path, bundles_root, out) + rows.append({**presentation, **identity}) monos, decoded, raw = _document_counts(document) closure = _closure_of(env, file_name) row = { diff --git a/src/ui/talk.py b/src/ui/talk.py index c1af96a..312da91 100644 --- a/src/ui/talk.py +++ b/src/ui/talk.py @@ -58,7 +58,7 @@ def resolve_script_class(obj, mono_index: dict) -> str: raw = obj.get_raw_data() if len(raw) < 28: return "?" - fid, pid = struct.unpack(" 2 * 1024 * 1024: + raise ValueError('font catalogue exceeds its document size limit') + files[f"revisions/{root['revision']}.json"] = data + files['latest.json'] = data + for name, content in files.items(): + path = output / name + if path.exists() and path.read_bytes() != content: + raise ValueError('font output differs; use a fresh publication directory') + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_bytes(content) + return root diff --git a/tests/check_distribution.py b/tests/check_distribution.py index 31711ec..3d3c2e2 100644 --- a/tests/check_distribution.py +++ b/tests/check_distribution.py @@ -56,7 +56,7 @@ def run(name, command): moly = scripts / ("moly.exe" if os.name == "nt" else "moly") run("install", [str(python), "-m", "pip", "install", str(wheel)]) run("dependencies", [str(python), "-m", "pip", "check"]) - for command in ([], ["pull"], ["extract"], ["fetch-apk"]): + for command in ([], ["pull"], ["extract"], ["fetch-apk"], ["source"], ["depot"], ["export", "text-runtime"]): run("help-" + (command[0] if command else "moly"), [str(moly), *command, "--help"]) runtime = r''' diff --git a/tests/test_asset_architecture_v2.py b/tests/test_asset_architecture_v2.py index b3efd0b..c89950c 100644 --- a/tests/test_asset_architecture_v2.py +++ b/tests/test_asset_architecture_v2.py @@ -710,6 +710,25 @@ def test_actor_export_plans_progress_receipts_and_backups_never_enter_runtime(as assert packages(before) == packages(after) +def test_bulk_gauge_document_and_textures_reach_the_published_catalogue(assets, tmp_path): + required = { + "ui-bulkharvest/gauge/fx_mysekai_bulk_harvest_gauge_charge.json": + b'{"geometry":null,"textures":["textures/pulse.png"]}', + "ui-bulkharvest/gauge/textures/pulse.png": b"synthetic gauge texture", + } + for name, data in required.items(): + path = assets / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + evidence = assets / "ui-bulkharvest/gauge/export-report.json" + evidence.write_bytes(b"private extraction evidence") + assert required.keys() <= runtime_paths(assets).keys() + catalog = build_groups(assets, tmp_path / "store", "7.0.0", region="jp") + published = {path for package in catalog["packages"] for path in package["paths"]} + assert required.keys() <= published + assert evidence.relative_to(assets).as_posix() not in published + + def test_verifier_shares_successful_blob_checks_only_within_one_call(assets, tmp_path, monkeypatch): import pack.verify as verifier out = tmp_path / "store" diff --git a/tests/test_audio_preserve.py b/tests/test_audio_preserve.py new file mode 100644 index 0000000..32fd00b --- /dev/null +++ b/tests/test_audio_preserve.py @@ -0,0 +1,70 @@ +"""Weather extraction must not erase a complete audio corpus registry.""" +import hashlib +import json +from pathlib import Path + +import pytest + +from phenomena import audio + + +def source(tmp_path): + root = tmp_path / 'audio'; (root / 'se').mkdir(parents=True) + raw = b'synthetic same-source cue archive' + (root / 'se/se.acb').write_bytes(raw) + (root / 'se/walk.ogg').write_bytes(b'walk') + (root / 'se/wind.ogg').write_bytes(b'wind') + prior = {'package': 'shared', 'archive': 'audio/se/se.acb', 'status': 'succeeded', + 'archiveSetRevision': 2, 'archiveSetCount': 1, 'archiveSetIndex': 0, + 'streams': [{'cue': 'walk', 'subsong': 1, 'ogg': 'audio/se/walk.ogg'}, + {'cue': 'wind', 'subsong': 2, 'ogg': 'audio/se/wind.ogg'}]} + untouched = {'package': 'voice', 'archive': 'audio/voice/voice.acb', 'streams': []} + (root / 'loop.json').write_text(json.dumps({'packages': [prior, untouched]})) + return root, raw, prior, untouched + + +def fresh(raw): + return {'package': 'shared', 'archive': 'audio/se/se.acb', 'status': 'succeeded', + 'sourceSha256': hashlib.sha256(raw).hexdigest(), + 'streams': [{'cue': 'wind', 'subsong': 2, 'ogg': 'audio/se/wind.ogg'}]} + + +def test_weather_subset_preserves_other_cues_and_other_banks(tmp_path, monkeypatch): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + library.packages = [fresh(raw)] + monkeypatch.setattr(audio, 'attach_sequences', lambda row, data: row.update( + sequenceTest={'cues': [s['cue'] for s in row['streams']], 'source': hashlib.sha256(data).hexdigest()})) + result = library.finish() + assert result['packages'][1] == untouched + combined = result['packages'][0] + assert combined['streams'] == prior['streams'] + assert combined['archiveSetRevision'] == 2 + assert combined['sequenceTest']['cues'] == ['walk', 'wind'] + assert combined['sourceSha256'] == hashlib.sha256(raw).hexdigest() + + +def test_changed_archive_fails_before_overwriting_audio(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + previous = (root / 'loop.json').read_bytes() + with pytest.raises(ValueError, match='identity changed'): + library.add('shared', 'se', b'another release') + assert (root / 'se/se.acb').read_bytes() == raw + assert (root / 'loop.json').read_bytes() == previous + + +def test_missing_preserved_stream_requires_repair(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio', preserve_existing=True) + library.packages = [fresh(raw)] + (root / 'se/walk.ogg').unlink() + with pytest.raises(ValueError, match='stream is missing'): + library.finish() + + +def test_default_library_still_replaces_explicit_whole_output(tmp_path): + root, raw, prior, untouched = source(tmp_path) + library = audio.Library(root, 'audio') + library.packages = [fresh(raw)] + assert library.finish()['packages'] == [fresh(raw)] diff --git a/tests/test_birthday_party_talks.py b/tests/test_birthday_party_talks.py index f28f8a9..5ad03a3 100644 --- a/tests/test_birthday_party_talks.py +++ b/tests/test_birthday_party_talks.py @@ -117,10 +117,9 @@ def test_an_absent_party_table_is_stated_not_silently_empty(tmp_path, monkeypatc assert "birthdayPartyScenarios" in block["reason"] -def test_the_party_block_is_the_only_key_the_change_adds(tmp_path, monkeypatch): - # The ordinary corpus is the document it always was, plus one key: a reader - # that held the top-level keys as a closed set would see exactly one new - # name, and no key it already had may be renamed, dropped or reordered. +def test_party_and_common_blocks_preserve_the_ordinary_corpus_keys(tmp_path, monkeypatch): + # Additional families have their own blocks; ordinary corpus keys stay + # present and ordered before them. monkeypatch.setattr(talks, "_text_assets", lambda bundle: _assets()) master = _master(tmp_path, [{"id": 1, "birthdayPartyId": 1, "seq": 1, "lua": "party_001"}]) @@ -128,7 +127,7 @@ def test_the_party_block_is_the_only_key_the_change_adds(tmp_path, monkeypatch): talks.extract_talks(master, "synthetic.bundle", out) document = json.loads(out.read_text(encoding="utf-8")) assert list(document) == ["version", "semantics", "units", "summary", - "birthdayParty"] + "birthdayParty", "commonSomeCharacter", "tutorial"] assert document["birthdayParty"]["semantics"]["table"].startswith( "birthdayPartyScenarios") @@ -175,7 +174,8 @@ def test_attach_leaves_every_other_key_alone(tmp_path, monkeypatch): after = json.loads(out.read_text(encoding="utf-8")) assert after["birthdayParty"]["status"] == "present" assert after["birthdayParty"]["talks"]["party_001"]["voices"] == ["voice_party_001_01_006"] - assert report["preservedKeys"] == ["version", "semantics", "units", "summary"] + assert report["preservedKeys"] == ["version", "semantics", "units", "summary", + "commonSomeCharacter", "tutorial"] # Byte for byte: with the block taken back out, the attached document is # the corpus the extraction wrote, minus that run's own absent block. del after["birthdayParty"] diff --git a/tests/test_bulk_harvest.py b/tests/test_bulk_harvest.py new file mode 100644 index 0000000..4abecaf --- /dev/null +++ b/tests/test_bulk_harvest.py @@ -0,0 +1,75 @@ +"""Runtime bindings must remain explicit and source-qualified.""" +from types import SimpleNamespace + +import pytest + +from core.gltf import GLB +from sites import bulk_harvest as bulk +from test_fixture_animations import _binding, _clip_tree + + +def _identity(pid): + return {"file": "CAB-synthetic", "pathId": str(pid)} + + +def _tables(): + names = ["CharacterAnimator", "Fixture1", "camera"] + tracks = [{"class": "AnimationTrack", "name": name, "asset": _identity(i)} + for i, name in enumerate(names, 1)] + source = {"timelines": [{"asset": _identity(9), "tracks": tracks}]} + targets = {"keyedClips": [{"track": t["asset"], "clipIndex": 0, + "target": {"sourceClip": _identity(20+i), "clipName": t["name"]+"_clip", + "targetPackage": "external" if i == 1 else bulk.PACKAGE}} + for i, t in enumerate(tracks)]} + director = {"timeline": _identity(9), "sceneBindings": [ + {"key": t["asset"], "value": None} for t in tracks]} + return source, targets, director + + +def test_bulk_streams_keep_the_declared_clip_package(): + source, targets, director = _tables() + selected = bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + assert selected["Fixture1"][0]["sourcePackage"] == "external" + assert selected["Fixture1"][0]["sourceClip"] == _identity(21) + director["sceneBindings"][1]["value"] = _identity(80) + with pytest.raises(ValueError, match="scene binding"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + + +def test_bulk_streams_refuse_unmodelled_or_missing_bindings(): + source, targets, director = _tables() + source["timelines"][0]["tracks"][2]["name"] = "second-camera" + with pytest.raises(ValueError, match="unsupported bulk"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + source, targets, director = _tables() + targets["keyedClips"][1]["target"].pop("sourceClip") + with pytest.raises(ValueError, match="unresolved clip"): + bulk.selected_tracks(source, targets, director, bulk.PACKAGE) + + +def test_camera_proxy_is_separate_and_has_no_source_object_identity(): + glb = GLB() + glb.g["nodes"] = [{"name": "source-root"}] + glb.g["scenes"][0]["nodes"] = [0] + tree = _clip_tree("camera", [_binding("", 1)], constant=[2.0, 3.0, 4.0]) + tree["m_MuscleClip"]["m_StopTime"] = 1.5 + records = SimpleNamespace(lookup=lambda _: (SimpleNamespace(tree=lambda _: tree), 1)) + clips = [{"name": "camera", "sourceClip": _identity(1)}] + bulk.append_camera(glb, records, clips) + assert glb.g["scenes"][0]["nodes"] == [0, 1] + assert glb.g["nodes"][1]["name"] == bulk.CAMERA_NODE + extras = glb.g["nodes"][1]["extras"] + assert "sourceObject" not in extras + assert extras["exportedProxy"]["cameraForwardCorrection"] == "runtime-y-pi" + assert clips[0]["sourceStopTime"] == 1.5 + assert glb.g["animations"][0]["channels"][0]["target"]["node"] == 1 + with pytest.raises(ValueError, match="already exists"): + bulk.append_camera(glb, records, clips) + + +def test_camera_rejects_a_non_root_curve(): + glb = GLB() + tree = _clip_tree("camera", [_binding("unbound", 1)], constant=[0.0, 0.0, 0.0]) + records = SimpleNamespace(lookup=lambda _: (SimpleNamespace(tree=lambda _: tree), 1)) + with pytest.raises(ValueError, match="unsupported bindings"): + bulk.append_camera(glb, records, [{"name": "camera", "sourceClip": _identity(1)}]) diff --git a/tests/test_character_animation_map.py b/tests/test_character_animation_map.py new file mode 100644 index 0000000..bfd2929 --- /dev/null +++ b/tests/test_character_animation_map.py @@ -0,0 +1,76 @@ +"""Controller identity must not widen a character's animation repertoire.""" +from types import SimpleNamespace + +import pytest + +from chara.characters import read_animation_controller + + +def ref(file_id, path_id): + return {"m_FileID": file_id, "m_PathID": path_id} + + +class Object: + def __init__(self, kind, path_id, tree, *, file="CAB-character", externals=()): + self.type = SimpleNamespace(name=kind) + self.path_id = path_id + self.assets_file = SimpleNamespace(name=file, externals=[SimpleNamespace(path=p) for p in externals]) + self.tree = tree + + def read_typetree(self): + return self.tree + + +def assets(clips, *, auxiliary=True): + controller = Object("AnimatorController", -123, {"m_AnimationClips": clips}, + externals=["archive:/CAB-motion/CAB-motion", "archive:\\CAB-other\\CAB-other"]) + animator = Object("Animator", 10, {"m_Controller": ref(0, -123)}) + objects = [animator, controller] + if auxiliary: + objects.insert(0, Object("Animator", 11, {"m_Controller": ref(0, 0)})) + return SimpleNamespace(objects=objects, deref=lambda owner, pointer: controller) + + +def test_actor_controller_keeps_source_membership_order_and_signed_ids(): + source = assets([ref(2, -(1 << 62)), ref(0, 41), ref(1, 41)]) + # A separately listed, unrelated controller must never supply the clips. + source.objects.append(Object("AnimatorController", 99, {"m_AnimationClips": [ref(0, 42)]})) + result = read_animation_controller(source) + assert result["animator"] == {"file": "CAB-character", "pathId": "10"} + assert result["controller"] == {"file": "CAB-character", "pathId": "-123"} + assert result["clips"] == [ + {"file": "CAB-other", "pathId": str(-(1 << 62))}, + {"file": "CAB-character", "pathId": "41"}, + {"file": "CAB-motion", "pathId": "41"}, + ] + # Same numeric path ID in another serialized file is a different clip. + index = {("CAB-character", "41"): "female_S", ("CAB-motion", "41"): "male_S", + ("CAB-other", str(-(1 << 62))): "idle_L"} + assert [index[(r["file"], r["pathId"])] for r in result["clips"]] == ["idle_L", "female_S", "male_S"] + + +def test_missing_controller_stays_null_but_empty_real_controller_stays_present(): + assert read_animation_controller(SimpleNamespace(objects=[])) is None + null = Object("Animator", 1, {"m_Controller": ref(0, 0)}) + assert read_animation_controller(SimpleNamespace(objects=[null])) is None + assert read_animation_controller(assets([]))["clips"] == [] + + +def test_ambiguous_animator_or_changed_controller_identity_is_refused(): + source = assets([]) + source.objects.append(Object("Animator", 12, {"m_Controller": ref(0, -123)})) + with pytest.raises(ValueError, match="one controller-bearing Animator"): + read_animation_controller(source) + source.objects.pop() + source.objects[-1].path_id = -124 + with pytest.raises(ValueError, match="identity mismatch"): + read_animation_controller(source) + + +@pytest.mark.parametrize("clips", [ + [ref(0, 0)], [ref(3, 1)], [ref(-1, 1)], [ref(True, 1)], [ref(1, True)], + [ref(0, 1 << 63)], [ref(1, 4), ref(1, 4)], [None], +]) +def test_invalid_or_duplicate_clip_references_do_not_produce_partial_maps(clips): + with pytest.raises(ValueError): + read_animation_controller(assets(clips)) diff --git a/tests/test_common_talks.py b/tests/test_common_talks.py new file mode 100644 index 0000000..03609ee --- /dev/null +++ b/tests/test_common_talks.py @@ -0,0 +1,75 @@ +"""Source-predicate probes over caller-supplied synthetic master tables. + +The source accepts two through five members, tests only the fixture-id +condition, and requires a matching association. Keep the boundary cases +alongside positive rows so dropping the whole family cannot satisfy them. +""" +import json + +from chara.common_talks import common_talks, select_common_talks +from core.master import Master +from phenomena.audio_corpus import voice_requests + + +def source(tmp_path): + def write(name, rows): + (tmp_path / f"{name}.json").write_text(json.dumps(rows), encoding="utf-8") + + groups = [] + for size in range(1, 6): + groups.append({"id": size, **{f"gameCharacterUnitId{i}": i for i in range(1, size + 1)}}) + write("mysekaiGameCharacterUnitGroups", groups) + kinds = ["mysekai_character_visit_count", "mysekai_fixture_id", "mysekai_fixture_tag_id"] + write("mysekaiCharacterTalkConditions", [ + {"id": i, "mysekaiCharacterTalkConditionType": kind, + "mysekaiCharacterTalkConditionTypeValue": 1} + for i, kind in enumerate(kinds, 1)]) + write("mysekaiCharacterTalkConditionGroups", [ + {"id": i, "groupId": i, "mysekaiCharacterTalkConditionId": i} for i in range(1, 4)]) + rows = [ + {"id": identity, "mysekaiGameCharacterUnitGroupId": size, + "mysekaiCharacterTalkConditionGroupId": condition, + "mysekaiSiteGroupId": 1, "mysekaiCharacterTalkTermId": 1, "lua": f"sample_{identity}"} + for identity, size, condition in [(1, 1, 1), (2, 2, 1), (3, 5, 1), + (4, 2, 2), (5, 2, 3), (6, 2, 1)]] + write("mysekaiCharacterTalks", rows) + write("mysekaiCharacterTalkSomeCharacterTalks", [ + {"id": identity, "mysekaiCharacterTalkId": identity, "mainGameCharacterUnitId": 2} + for identity in range(1, 6)]) + write("mysekaiCharacterTalkPreActions", []) + write("mysekaiCharacterTalkTweets", []) + return Master(tmp_path) + + +def test_native_membership_boundaries_and_exact_fixture_predicate(tmp_path): + rows, report = select_common_talks(source(tmp_path)) + assert [row["talk"]["id"] for row in rows] == [2, 3, 5] + assert rows[1]["unitIds"] == [1, 2, 3, 4, 5] + assert report["withoutAssociation"] == [6] + + +def test_shared_parser_preserves_both_speakers_and_routes_common_voice_banks(tmp_path): + scripts = {f"sample_{i}.lua": f'''label("Alpha") +voice("talk", "voice_mysekai_sample_{i}_01_001", Characters.Alpha) +text("First") +wait_click() +label("Beta") +voice("talk", "voice_mysekai_sample_{i}_02_002", Characters.Beta) +text("Second") +wait_click() +''' for i in [2, 3, 5]} + block, report = common_talks(source(tmp_path), scripts, {"Characters": {"Alpha": 1, "Beta": 2}}, {}) + assert report["talks"] == 3 + assert [s["who"] for s in block["talks"][0]["steps"] if s["op"] == "voice"] == [1, 2] + assert block["talks"][0]["someCharacterTalks"][0]["mainGameCharacterUnitId"] == 2 + assert "mainUnitId" not in block["talks"][0] + requests, cues = voice_requests({"commonSomeCharacter": block}) + assert len(requests) == 3 + assert len(cues) == 6 + + +def test_unrecognized_source_call_is_reported_before_export(tmp_path): + import pytest + scripts = {"sample_2.lua": 'new_source_operation("value")'} + with pytest.raises(ValueError, match="unknown script operations"): + common_talks(source(tmp_path), scripts, {}, {}) diff --git a/tests/test_editor_environment_layout.py b/tests/test_editor_environment_layout.py new file mode 100644 index 0000000..71e49f6 --- /dev/null +++ b/tests/test_editor_environment_layout.py @@ -0,0 +1,25 @@ +"""The environment cell follows the same serialized list reference as the client.""" +import struct + +from ui import editor_layout, prefab_layout, talk + + +def test_environment_cell_keeps_list_metrics_and_the_two_references(): + reader = talk.Reader(struct.pack(" 2 * 1024 * 1024: + raise ValueError('font catalogue exceeds document limit') + # Browser/Go consumers decode UTF-8, whereas json.loads(bytes) also accepts + # UTF-16/32. Reject encodings that would publish successfully but not load. + root = json.loads(data.decode('utf-8', errors='strict')) + + def is_hash(value): + return isinstance(value, str) and HASH.fullmatch(value) + + if (not isinstance(root, dict) or root.get('schema') != 'moly-text-fonts/1' + or not is_hash(root.get('revision')) + or not isinstance(root.get('baseFonts'), dict) or not 1 <= len(root['baseFonts']) <= 16 + or not isinstance(root.get('fonts'), list) or not 1 <= len(root['fonts']) <= 8): + raise ValueError('invalid font catalogue') + identity = {k: v for k, v in root.items() if k != 'revision'} + canonical = (json.dumps(identity, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode() + if hashlib.sha256(canonical).hexdigest() != root['revision']: + raise ValueError('font catalogue revision mismatch') + + def coverage(rows): + if not isinstance(rows, list) or not rows or len(rows) > 65536: + raise ValueError('invalid font coverage ranges') + previous = -1 + for row in rows: + if (not isinstance(row, list) or len(row) != 2 or any(type(n) is not int for n in row) + or not previous < row[0] <= row[1] <= 0x10FFFF + or (row[0] <= 0xDFFF and row[1] >= 0xD800)): + raise ValueError('invalid or overlapping font coverage') + previous = row[1] + + for sha, entry in root['baseFonts'].items(): + if not is_hash(sha) or not isinstance(entry, dict): + raise ValueError('invalid base font identity') + coverage(entry.get('coverage')) + root_ref = {'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data)} + files = {'latest.json': root_ref, f"revisions/{root['revision']}.json": root_ref} + + def reference(ref, suffix, maximum): + if (not isinstance(ref, dict) or not is_hash(ref.get('sha256')) + or ref.get('path') != f"objects/{ref['sha256']}.{suffix}" + or type(ref.get('bytes')) is not int or not 0 < ref['bytes'] <= maximum): + raise ValueError('invalid immutable font object') + expected = {'sha256': ref['sha256'], 'bytes': ref['bytes']} + if ref['path'] in files and files[ref['path']] != expected: + raise ValueError('conflicting font object reference') + files[ref['path']] = expected + + seen = set() + for entry in root['fonts']: + if not isinstance(entry, dict): + raise ValueError('invalid supplemental font') + coverage(entry.get('coverage')) + reference(entry, 'ttf', 32 * 1024 * 1024) + reference(entry.get('license'), 'txt', 65536) + if entry['sha256'] in seen: + raise ValueError('duplicate supplemental font') + seen.add(entry['sha256']) + return root, files + + +def prepare(publication, output, *, object_prefix, url_prefix): + publication = Path(publication).resolve() + output = Path(output).resolve() + data = (publication / 'latest.json').read_bytes() + root, references = parse_catalog(data) + files = {} + for name, ref in references.items(): + path = (publication / name).resolve() + if not path.is_relative_to(publication): + raise ValueError('font object escapes publication') + raw = path.read_bytes() + if len(raw) != ref['bytes'] or hashlib.sha256(raw).hexdigest() != ref['sha256']: + raise ValueError('font object integrity mismatch') + files[name] = raw + if ((object_prefix and not object_prefix.endswith('/')) or object_prefix.startswith('/') or not url_prefix.startswith('/') + or not url_prefix.endswith('/') or any(part in {'.', '..'} for value in [object_prefix, url_prefix] for part in value.split('/')) + or any(char in object_prefix + url_prefix for char in '\\%?#:')): + raise ValueError('publication prefixes must be directories') + rows = [] + for name, raw in sorted(files.items()): + logical = 'fonts/' + name + target = output / 'staging' / logical + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != raw: + raise ValueError('staging bytes differ; choose a fresh directory') + if not target.exists(): + target.write_bytes(raw) + sha = hashlib.sha256(raw).hexdigest() + rows.append({'path': logical, 'urlPath': url_prefix + logical, 'objectKey': object_prefix + logical, + 'source': str(target), 'sourceRaw': str(target), 'staging': str(target), + 'sha256': sha, 'decodedSha256': sha, 'bytes': len(raw), 'decodedBytes': len(raw), + 'contentEncoding': 'identity', 'contentType': content_type(name), + 'cacheControl': cache_control(name)}) + result = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, + 'urlPrefix': url_prefix, 'snapshots': [], 'fontCatalog': {'path': 'fonts/latest.json', + 'revision': root['revision'], 'sha256': hashlib.sha256(data).hexdigest()}, + 'files': rows, 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows)}} + (output / 'upload-manifest.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--publication', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--object-prefix', default='sekai-extra-assets/') + parser.add_argument('--url-prefix', default='/moly/') + args = parser.parse_args() + result = prepare(args.publication, args.output, object_prefix=args.object_prefix, url_prefix=args.url_prefix) + print(json.dumps({'fontCatalog': result['fontCatalog'], 'totals': result['totals']})) diff --git a/tools/deployment/prepare_resources.py b/tools/deployment/prepare_resources.py new file mode 100644 index 0000000..f412105 --- /dev/null +++ b/tools/deployment/prepare_resources.py @@ -0,0 +1,95 @@ +"""Prepare one immutable resource catalogue without an app release or pointer.""" +from pathlib import Path +import argparse +import os +import sys + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parents[1] / 'src')) +from pack.groups import runtime_paths +from pack.http_delivery import choose_representation +from prepare_deployment import browser_membership, content_type, delivery_profile, write_json +from resource_publication import catalog_closure, digest, CACHE_CONTROL +from verified_representations import VerifiedRepresentations + + +def prepare(source, store, catalog_id, output, *, object_prefix='', url_prefix='/moly/', encoding='best', stage=False, verified_representations=None): + source, store, output = Path(source).resolve(strict=True), Path(store).resolve(strict=True), Path(output).resolve() + for left, right in [(source, store), (source, output), (store, output)]: + if left.is_relative_to(right) or right.is_relative_to(left): + raise ValueError('source, store and staging directories must be disjoint') + if ((object_prefix and (not object_prefix.endswith('/') or object_prefix.startswith('/') + or any(part in {'', '.', '..'} for part in object_prefix[:-1].split('/')))) + or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for part in url_prefix.split('/')) + or any(ord(c) < 32 or ord(c) == 127 or c in '\\%?#:' for c in object_prefix + url_prefix)): + raise ValueError('invalid publication prefix') + identity, objects, entries, verified = catalog_closure(store, catalog_id) + members, audit = browser_membership(source, identity['region'], identity['version'], runtime_paths) + if audit['missing'] or audit['browserBaseMismatch'] or members.keys() != entries.keys(): + raise ValueError('resource catalogue does not hold the complete browser membership') + for name, path in members.items(): + data = path.read_bytes() + if len(data) != entries[name]['bytes'] or digest(data) != entries[name]['content_sha256']: + raise ValueError('resource catalogue content differs from the supplied source: ' + name) + profile = delivery_profile(encoding, True) + reuse = VerifiedRepresentations(verified_representations) + rows, invalid_sidecars = [], [] + for name, item in sorted(objects.items()): + logical = 'asset-store/' + name + media_type = content_type(logical, item['source']) + reused = reuse.select(item['source'], logical, object_prefix + logical, media_type) + representation = reused.representation if reused else choose_representation(item['source'], logical, output / 'representations', mode=encoding) + if representation.decoded_sha256 != item['sha256'] or representation.decoded_bytes != item['bytes']: + raise ValueError('store changed while preparing HTTP representations') + row = {'path': logical, 'objectKey': object_prefix + logical, 'urlPath': url_prefix + logical, + 'contentType': content_type(logical, item['source']), 'cacheControl': CACHE_CONTROL, + **representation.manifest_fields} + if reused: + reused.apply(row) + invalid_sidecars.extend(representation.invalid_sidecars) + if stage: + target = output / 'staging' / representation.encoding / logical + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + if target.read_bytes() != representation.body: + raise ValueError('staging content changed; use a fresh directory') + else: + try: + os.link(representation.source, target) + except OSError: + with target.open('xb') as stream: + stream.write(representation.body) + row['staging'] = str(target) + row['source'] = str(target) + rows.append(row) + manifest = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, + 'urlPrefix': url_prefix, 'snapshots': [], 'resourceCatalog': identity, 'files': rows, + 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows), + 'decodedBytes': sum(row['decodedBytes'] for row in rows)}, 'deliveryProfile': profile} + write_json(output / 'audit.json', {'resourceCatalog': identity, 'source': audit, + 'verifiedStore': verified, 'invalidSidecars': invalid_sidecars, 'verifiedRepresentations': reuse.audit()}) + write_json(output / 'upload-manifest.json', manifest) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--store', type=Path, required=True) + parser.add_argument('--catalog', required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--object-prefix', default='') + parser.add_argument('--url-prefix', default='/moly/') + parser.add_argument('--encoding', choices=['identity', 'gzip', 'best'], default='best') + parser.add_argument('--verified-representations', type=Path, help='Explicit local index of verified immutable origin bodies and headers') + parser.add_argument('--stage', action='store_true') + args = parser.parse_args() + result = prepare(args.source, args.store, args.catalog, args.output, object_prefix=args.object_prefix, + url_prefix=args.url_prefix, encoding=args.encoding, stage=args.stage, verified_representations=args.verified_representations) + import json + print(json.dumps({'resourceCatalog': result['resourceCatalog'], 'totals': result['totals']})) + + +if __name__ == '__main__': + main() diff --git a/tools/deployment/prepare_translations.py b/tools/deployment/prepare_translations.py new file mode 100644 index 0000000..43c538b --- /dev/null +++ b/tools/deployment/prepare_translations.py @@ -0,0 +1,107 @@ +"""Prepare an independent translation publication for the existing uploaders. + +No resource snapshot is read or changed. This command performs no cloud writes. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re + +HASH = re.compile(r'[a-f0-9]{64}') + + +def prepare(publication, output, *, object_prefix, url_prefix): + publication = Path(publication).resolve() + output = Path(output).resolve() + root_bytes = (publication / 'latest.json').read_bytes() + if len(root_bytes) > 2 * 1024 * 1024: + raise ValueError('translation root exceeds limit') + root = json.loads(root_bytes) + if (root.get('schema') != 'moly-text-publication/1' or root.get('region') not in {'jp', 'cn', 'tw', 'en', 'kr'} + or root.get('locale') not in {'zh-CN', 'zh-TW', 'en-US', 'ja-JP', 'ko-KR'} + or not HASH.fullmatch(root.get('revision', '')) or root.get('routing') != 'sha256-utf8-prefix-2' + or not isinstance(root.get('buckets'), dict) or len(root['buckets']) > 256 + or len(root.get('searchBuckets', {})) > 16 or root.get('fonts')): + raise ValueError('invalid translation publication identity') + base = f"translations/{root['region']}/{root['locale']}/" + files = {'latest.json': root_bytes, f"revisions/{root['revision']}.json": root_bytes} + + def read(ref): + if (not isinstance(ref, dict) or not HASH.fullmatch(ref.get('sha256', '')) + or ref.get('path') != f"objects/{ref['sha256']}.json" + or type(ref.get('bytes')) is not int or not 0 < ref['bytes'] <= 2 * 1024 * 1024): + raise ValueError('invalid translation object reference') + path = (publication / ref['path']).resolve() + if not path.is_relative_to(publication): + raise ValueError('translation path escaped publication') + data = path.read_bytes() + if len(data) != ref['bytes'] or hashlib.sha256(data).hexdigest() != ref['sha256']: + raise ValueError('translation object integrity mismatch') + files[ref['path']] = data + return json.loads(data) + + for prefix, ref in root['buckets'].items(): + if not re.fullmatch('[a-f0-9]{2}', prefix): + raise ValueError('invalid translation route prefix') + routes = read(ref) + if routes.get('schema') != 'moly-text-translation-routes/1': + raise ValueError('invalid translation routes') + for key, entry in routes['scripts'].items(): + if hashlib.sha256(key.encode()).hexdigest()[:2] != prefix: + raise ValueError('translation script in wrong bucket') + overlay = read(entry) + if (overlay.get('schema') != 'moly-text-overlay/1' or overlay.get('sourceRegion') != root['region'] + or overlay.get('locale') != root['locale']): + raise ValueError('translation overlay scope mismatch') + for prefix, ref in root.get('searchBuckets', {}).items(): + search = read(ref) + if (not re.fullmatch('[a-f0-9]', prefix) or search.get('schema') != 'moly-text-translation-search/1' + or any(hashlib.sha256(key.encode()).hexdigest()[:1] != prefix for key in search['scripts'])): + raise ValueError('invalid translation search shard') + if (publication / f"revisions/{root['revision']}.json").read_bytes() != root_bytes: + raise ValueError('archived publication differs from latest') + if ((object_prefix and not object_prefix.endswith('/')) or object_prefix.startswith('/') or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for prefix in [object_prefix, url_prefix] for part in prefix.split('/')) + or any(char in object_prefix + url_prefix for char in '\\%?#:')): + raise ValueError('publication prefixes must be directories') + output.mkdir(parents=True, exist_ok=True) + rows = [] + for name, data in sorted(files.items()): + logical = base + name + target = output / 'staging' / logical + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != data: + raise ValueError('staging bytes differ; choose a fresh output directory') + if not target.exists(): + target.write_bytes(data) + sha = hashlib.sha256(data).hexdigest() + rows.append({'path': logical, 'urlPath': url_prefix + logical, 'objectKey': object_prefix + logical, + 'source': str(target), 'sourceRaw': str(target), 'staging': str(target), + 'sha256': sha, 'decodedSha256': sha, 'bytes': len(data), 'decodedBytes': len(data), + 'contentEncoding': 'identity', 'contentType': 'application/json; charset=utf-8', + 'cacheControl': 'public, max-age=15, must-revalidate' if name == 'latest.json' + else 'public, max-age=31536000, immutable'}) + result = {'schema': 'moly-object-upload/1', 'deliveryMode': 'fixed', 'objectPrefix': object_prefix, + 'urlPrefix': url_prefix, 'snapshots': [], 'translation': {'path': base + 'latest.json', + 'revision': root['revision'], 'sha256': hashlib.sha256(root_bytes).hexdigest()}, + 'totals': {'files': len(rows), 'bytes': sum(row['bytes'] for row in rows)}, 'files': rows} + (output / 'upload-manifest.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--publication', required=True, type=Path, help='Locale directory containing latest.json') + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--object-prefix', default='sekai-extra-assets/') + parser.add_argument('--url-prefix', default='/moly/') + args = parser.parse_args() + result = prepare(args.publication, args.output, object_prefix=args.object_prefix, url_prefix=args.url_prefix) + print(json.dumps({'translation': result['translation'], 'totals': result['totals']})) + + +if __name__ == '__main__': + main() diff --git a/tools/deployment/resource_publication.py b/tools/deployment/resource_publication.py new file mode 100644 index 0000000..fadbf53 --- /dev/null +++ b/tools/deployment/resource_publication.py @@ -0,0 +1,155 @@ +"""The immutable asset catalogue closure, independent of an application release.""" +from pathlib import Path +import hashlib +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'src')) +from core.jp import public_release_provenance +from pack.identity import canonical_bytes +from pack.limits import MAX_RUNTIME_BLOB_BYTES, MAX_RUNTIME_DOCUMENT_BYTES +from pack.verify import strict_json, verify_catalog_data, _decode_content + +HASH = re.compile(r'[a-f0-9]{64}') +CACHE_CONTROL = 'public, max-age=31536000, immutable, no-transform' + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def read_store_file(store, relative, maximum): + path = store / relative + if (not relative or relative.startswith('/') or '\\' in relative or ':' in relative + or any(part in {'', '.', '..'} for part in relative.split('/'))): + raise ValueError('invalid store object path') + for component in (path, *path.parents): + if component == store: + break + if component.is_symlink() or getattr(component, 'is_junction', lambda: False)(): + raise ValueError('linked store object path') + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(store): + raise ValueError('store object escaped root') + with resolved.open('rb') as stream: + data = stream.read(maximum + 1) + if len(data) > maximum: + raise ValueError('store object exceeds runtime budget') + return resolved, data + + +def catalog_closure(store, catalog_id): + """Verify the catalogue, package graph, source identity and all stored blobs.""" + store = Path(store).resolve(strict=True) + if not isinstance(catalog_id, str) or not HASH.fullmatch(catalog_id): + raise ValueError('invalid resource catalogue hash') + name = f'catalogs/{catalog_id}.json' + path, raw = read_store_file(store, name, MAX_RUNTIME_DOCUMENT_BYTES) + catalog = strict_json(raw) + if (digest(raw) != catalog_id or catalog.get('schema') != 'moly-asset-packs/2' + or catalog.get('region') not in {'jp', 'cn', 'tw', 'en', 'kr'} + or not re.fullmatch(r'\d+\.\d+\.\d+', catalog.get('version', '')) + or raw != canonical_bytes(catalog)): + raise ValueError('invalid resource catalogue identity') + if public_release_provenance(catalog.get('provenance')) != catalog.get('provenance'): + raise ValueError('resource catalogue provenance is not portable') + errors, info = verify_catalog_data(catalog, store) + if errors: + raise ValueError('invalid resource catalogue closure: ' + '; '.join(errors[:8])) + objects = {name: {'source': path, 'sha256': catalog_id, 'bytes': len(raw)}} + entries = {} + for package in catalog['packages']: + name = package['manifest'] + path, raw = read_store_file(store, name, MAX_RUNTIME_DOCUMENT_BYTES) + if digest(raw) != Path(name).stem: + raise ValueError('package changed during resource verification') + objects[name] = {'source': path, 'sha256': digest(raw), 'bytes': len(raw)} + for entry in strict_json(raw)['entries']: + # The v2 native and Web readers share this representation contract. + # The archive verifier also accepts legacy Brotli forms, but those + # cannot be published as a usable runtime catalogue. + if (entry['codec'] not in {'identity', 'gzip'} + or entry['http_encoding'] != 'identity' or entry['xf'] is not None): + raise ValueError('resource catalogue uses a representation unsupported by native/Web runtime') + entries[entry['path']] = entry + name = 'blobs/' + entry['blob'] + path, raw = read_store_file(store, name, MAX_RUNTIME_BLOB_BYTES) + if len(raw) != entry['blob_bytes'] or digest(raw) != entry['blob_sha256']: + raise ValueError('blob changed during resource verification') + objects[name] = {'source': path, 'sha256': entry['blob_sha256'], 'bytes': entry['blob_bytes']} + source_entry = entries.get('source.json') + provenance = catalog.get('provenance', {}).get('runtime_source') + if source_entry is None or not isinstance(provenance, dict): + raise ValueError('resource catalogue requires its runtime source document') + _, source_blob = read_store_file(store, 'blobs/' + source_entry['blob'], MAX_RUNTIME_BLOB_BYTES) + source_raw = _decode_content(source_blob, source_entry['codec'], source_entry['http_encoding'], source_entry['bytes']) + if strict_json(source_raw) != provenance: + raise ValueError('catalogue provenance differs from the packaged source document') + source_identity = provenance.get('source', {}) + if (source_identity.get('region') != catalog['region'] + or source_identity.get('appVersion') != catalog['version']): + raise ValueError('packaged source region/version differs from its resource catalogue') + identity = {'path': f'asset-store/catalogs/{catalog_id}.json', 'sha256': catalog_id, + 'bytes': objects[f'catalogs/{catalog_id}.json']['bytes'], + 'region': catalog['region'], 'version': catalog['version'], + 'provenance': catalog['provenance']} + return identity, objects, entries, info + + +def validate_upload(doc): + """Revalidate actual sources and representations before selecting any upload.""" + from prepare_deployment import content_type + identity = doc['resourceCatalog'] + root_rows = [row for row in doc['files'] if row['path'] == identity['path']] + if len(root_rows) != 1: + raise ValueError('resource catalogue row is absent or duplicated') + raw_path = Path(root_rows[0]['sourceRaw']).absolute() + if raw_path.parent.name != 'catalogs' or raw_path.name != identity['sha256'] + '.json': + raise ValueError('resource catalogue source is not its addressed store object') + store = raw_path.parent.parent.resolve(strict=True) + actual, objects, _, _ = catalog_closure(store, identity['sha256']) + if actual != identity: + raise ValueError('resource catalogue identity or provenance changed') + rows = {row['path']: row for row in doc['files']} + expected = {'asset-store/' + name for name in objects} + if rows.keys() != expected: + raise ValueError('resource publication object closure mismatch') + url_prefix = doc.get('urlPrefix') + if (not isinstance(url_prefix, str) or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for part in url_prefix.split('/')) + or any(ord(c) < 32 or ord(c) == 127 or c in '\\%?#:' for c in url_prefix)): + raise ValueError('invalid resource URL prefix') + checked = set() + for name, expected in objects.items(): + logical = 'asset-store/' + name + row = rows[logical] + media_type = content_type(logical, expected['source']) + reused = None + if 'representationReuse' in row: + from verified_representations import validate_reused_row + reused = validate_reused_row(row, expected['source'], media_type) + if (Path(row['sourceRaw']).absolute() != expected['source'] + or row['decodedSha256'] != expected['sha256'] or row['decodedBytes'] != expected['bytes'] + or (reused is None and row['bytes'] > expected['bytes']) + or row['contentType'] != content_type(logical, expected['source']) + or (reused is None and row['cacheControl'] != CACHE_CONTROL) or row.get('urlPath') != url_prefix + logical): + raise ValueError('resource object metadata differs from its catalogue') + for alias in [row.get('source'), *([row['staging']] if 'staging' in row else [])]: + if not isinstance(alias, str) or not alias: + raise ValueError('invalid resource object source') + path = Path(alias).resolve(strict=True) + key = (path, row['sha256'], row['bytes'], expected['sha256'], expected['bytes'], row['contentEncoding']) + if key in checked: + continue + with path.open('rb') as stream: + body = stream.read(row['bytes'] + 1) + if len(body) != row['bytes'] or digest(body) != row['sha256']: + raise ValueError('local resource representation changed') + # The HTTP representation expands to the stored blob/package bytes. + # A pack codec, if present, is subsequently decoded by the runtime. + encoding = row['contentEncoding'] + decoded = _decode_content(body, 'gzip' if encoding == 'gzip' else 'identity', + 'br' if encoding == 'br' else 'identity', expected['bytes']) + if len(decoded) != expected['bytes'] or digest(decoded) != expected['sha256']: + raise ValueError('resource HTTP representation differs from the addressed object') + checked.add(key) diff --git a/tools/deployment/upload_oss.py b/tools/deployment/upload_oss.py index 7bba12c..9ef576d 100644 --- a/tools/deployment/upload_oss.py +++ b/tools/deployment/upload_oss.py @@ -24,7 +24,14 @@ REGION = 'cn-shanghai' ENDPOINT = 'https://oss-cn-shanghai.aliyuncs.com' PREFIX = 'moly/alpha/' -MUTABLE = {'manifest.json', 'cache-worker.mjs', 'game/manifest.json'} +MUTABLE = {'manifest.json', 'cache-worker.mjs', 'game/manifest.json', 'fonts/latest.json'} +TRANSLATION_ROOT = re.compile(r'translations/(jp|cn|tw|en|kr)/(zh-CN|zh-TW|en-US|ja-JP|ko-KR)/latest\.json') + + +def mutable_path(value): + if value not in MUTABLE and not TRANSLATION_ROOT.fullmatch(value): + raise argparse.ArgumentTypeError('expected a known publication pointer') + return value IDENTIFIER = re.compile(r'^[a-z0-9][a-z0-9._-]{0,95}$') SHA256 = re.compile(r'^[a-f0-9]{64}$') @@ -144,7 +151,7 @@ def parse_args(): p.add_argument('--manifest', type=Path, required=True) p.add_argument('--scope', choices=['stable', 'release', 'descriptors', 'mutable'], default='stable') p.add_argument('--release-id', help='Required explicit release ID for --scope release') - p.add_argument('--mutable-path', choices=sorted(MUTABLE), action='append', default=[]) + p.add_argument('--mutable-path', type=mutable_path, action='append', default=[]) p.add_argument('--allow-mutable-update', action='store_true') p.add_argument('--expect-previous-sha256', help='Required to replace an existing mutable object; one mutable path per run') p.add_argument('--threads', type=int, default=16, choices=range(1, 17), metavar='1..16') @@ -159,6 +166,75 @@ def parse_args(): return p.parse_args() +def validate_font_publication(doc): + # Preparation is not a trust boundary: manifests and staging paths can + # change before execution. Reuse its catalogue contract, then bind every + # declared input and the actual upload stream to the full root closure. + from prepare_fonts import parse_catalog, content_type, cache_control + + catalog = doc['fontCatalog'] + rows = {row['path']: row for row in doc['files']} + root_row = rows.get(catalog['path']) + if root_row is None: + raise SafeFailure('font_pointer_identity_mismatch') + try: + with Path(root_row['sourceRaw']).open('rb') as stream: + raw = stream.read(2 * 1024 * 1024 + 1) + root, references = parse_catalog(raw) + except (OSError, ValueError, KeyError, TypeError) as error: + raise SafeFailure('invalid_local_font_catalogue', errorType=type(error).__name__) from None + if (root['revision'] != catalog['revision'] + or references['latest.json']['sha256'] != catalog['sha256']): + raise SafeFailure('font_pointer_identity_mismatch') + expected_paths = {'fonts/' + name for name in references} + if rows.keys() != expected_paths: + raise SafeFailure('font_publication_closure_mismatch', + missing=sorted(expected_paths - rows.keys()), + extra=sorted(rows.keys() - expected_paths)) + url_prefix = doc.get('urlPrefix') + if (not isinstance(url_prefix, str) or not url_prefix.startswith('/') or not url_prefix.endswith('/') + or any(part in {'.', '..'} for part in url_prefix.split('/')) + or any(char in url_prefix for char in '\\%?#:') + or any(ord(char) < 32 or ord(char) == 127 for char in url_prefix)): + raise SafeFailure('invalid_font_url_prefix') + verified = set() + for name, expected in references.items(): + logical = 'fonts/' + name + row = rows[logical] + if (row['sha256'] != expected['sha256'] or row['decodedSha256'] != expected['sha256'] + or row['bytes'] != expected['bytes'] or row['decodedBytes'] != expected['bytes'] + or row['contentEncoding'] != 'identity' or row['contentType'] != content_type(name) + or row['cacheControl'] != cache_control(name) + or row.get('urlPath') != url_prefix + logical): + raise SafeFailure('font_object_metadata_mismatch', path=logical) + # All aliases are checked; sourceRaw must not validate one document + # while staging/source supplies a different body to put_one(). + aliases = [row.get('sourceRaw'), row.get('source')] + if 'staging' in row: + aliases.append(row['staging']) + for alias in aliases: + if not isinstance(alias, str) or not alias: + raise SafeFailure('invalid_font_object_source', path=logical) + try: + path = Path(alias).resolve(strict=True) + identity = (path, expected['sha256'], expected['bytes']) + if identity in verified: + continue + digest, size = hashlib.sha256(), 0 + with path.open('rb') as stream: + while block := stream.read(1024 * 1024): + size += len(block) + if size > expected['bytes']: + raise SafeFailure('local_font_object_changed', path=logical) + digest.update(block) + if size != expected['bytes'] or digest.hexdigest() != expected['sha256']: + raise SafeFailure('local_font_object_changed', path=logical) + verified.add(identity) + except OSError as error: + raise SafeFailure('cannot_read_font_object', path=logical, + errorType=type(error).__name__) from None + + def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',)): try: doc = json.loads(args.manifest.read_bytes()) @@ -169,9 +245,66 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) if doc.get('deliveryMode', 'fixed') != 'fixed': raise SafeFailure('unsupported_delivery_mode') snapshots = set(doc.get('snapshots', [])) - if not snapshots or any(not IDENTIFIER.fullmatch(s) for s in snapshots): + translation = doc.get('translation') + font_catalog = doc.get('fontCatalog') + resource_catalog = doc.get('resourceCatalog') + if resource_catalog is not None: + if (not isinstance(resource_catalog, dict) or snapshots or translation is not None + or font_catalog is not None or doc.get('release') is not None + or not SHA256.fullmatch(resource_catalog.get('sha256', '')) + or resource_catalog.get('path') != f"asset-store/catalogs/{resource_catalog['sha256']}.json"): + raise SafeFailure('invalid_resource_catalogue_publication') + if args.scope != 'stable' or args.limit is not None: + raise SafeFailure('resource_catalogue_requires_complete_stable_scope') + if font_catalog is not None: + if (not isinstance(font_catalog, dict) or snapshots or translation is not None + or font_catalog.get('path') != 'fonts/latest.json' + or not SHA256.fullmatch(font_catalog.get('revision', '')) + or not SHA256.fullmatch(font_catalog.get('sha256', ''))): + raise SafeFailure('invalid_font_publication') + if translation is not None: + if (not isinstance(translation, dict) or snapshots + or not TRANSLATION_ROOT.fullmatch(translation.get('path', '')) + or not SHA256.fullmatch(translation.get('revision', '')) + or not SHA256.fullmatch(translation.get('sha256', ''))): + raise SafeFailure('invalid_translation_publication') + translation_base = translation['path'].removesuffix('latest.json') + else: + translation_base = None + if (not snapshots and translation is None and font_catalog is None and resource_catalog is None) or any(not IDENTIFIER.fullmatch(s) for s in snapshots): raise SafeFailure('invalid_snapshot_identities') - if args.scope == 'release' and (args.release_id != doc.get('release') or not IDENTIFIER.fullmatch(args.release_id or '')): + releases = doc.get('releases', [doc['release']] if doc.get('release') else []) + if (not isinstance(releases, list) or len(releases) > 2 + or any(not isinstance(value, str) or not IDENTIFIER.fullmatch(value) for value in releases) + or len(set(releases)) != len(releases) + or (releases and doc.get('release') is None) + or (doc.get('release') is not None and doc['release'] not in releases) + or (releases and (resource_catalog is not None or font_catalog is not None or translation is not None))): + raise SafeFailure('invalid_release_identities') + if len(releases) > 1: + # A second release must be the actual game pointer target, not an + # arbitrary directory added to the upload inventory by an operator. + pointers = {} + for logical in ('manifest.json', 'game/manifest.json'): + rows = [row for row in doc.get('files', []) if row.get('path') == logical] + if len(rows) != 1: + raise SafeFailure('missing_release_pointer', path=logical) + try: + raw = Path(rows[0]['sourceRaw']).read_bytes() + if hashlib.sha256(raw).hexdigest() != rows[0]['decodedSha256']: + raise SafeFailure('local_descriptor_hash_mismatch', path=logical) + pointers[logical] = json.loads(raw) + except (OSError, KeyError, TypeError, ValueError) as error: + raise SafeFailure('cannot_read_release_pointer', path=logical) from None + primary = pointers['manifest.json'].get('release', {}).get('id') + game = pointers['game/manifest.json'] + game_id = game.get('release', {}).get('id') + if (primary != doc.get('release') or releases != list(dict.fromkeys([primary, game_id])) + or game.get('publisher') != 'moly-release-artifact-v1' or game.get('schemaVersion') != 1 + or game.get('prefix') != doc.get('urlPrefix') + or game.get('release', {}).get('module') != f"{doc.get('urlPrefix')}releases/{game_id}/game-boot.mjs"): + raise SafeFailure('release_pointers_do_not_match_manifest') + if args.scope == 'release' and args.release_id not in releases: raise SafeFailure('explicit_release_id_must_match_manifest') if args.scope == 'mutable': if not args.allow_mutable_update or len(args.mutable_path) != 1: @@ -187,6 +320,13 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected, seen = [], set() for row in doc.get('files', []): logical = row.get('path', '') + if resource_catalog is not None and not re.fullmatch(r'asset-store/(?:(?:catalogs|packages)/[a-f0-9]{64}\.json|blobs/[a-f0-9]{2}/[a-f0-9]{64}\.(?:bin|gzz|brz|br))', logical): + raise SafeFailure('resource_catalogue_contains_unrelated_object') + if font_catalog is not None and not re.fullmatch(r'fonts/(?:latest\.json|revisions/[a-f0-9]{64}\.json|objects/[a-f0-9]{64}\.(?:ttf|txt))', logical): + raise SafeFailure('font_manifest_contains_unrelated_object') + if translation_base and (not logical.startswith(translation_base) or not re.fullmatch( + r'(?:latest\.json|(?:objects|revisions)/[a-f0-9]{64}\.json)', logical[len(translation_base):])): + raise SafeFailure('translation_manifest_contains_unrelated_object') parts = PurePosixPath(logical).parts # This is an object key, not a URL. S3 permits a literal '#'; URL # consumers must percent-encode that segment when constructing a @@ -220,8 +360,10 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) is_snapshot = len(parts) >= 3 and parts[0] == 'snapshots' and parts[1] in snapshots # The shared content-addressed stores are immutable like snapshot payloads. shared_store = len(parts) >= 2 and parts[0] in {'asset-store', 'catalog-store'} - choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog'} and len(parts) >= 4) - or shared_store)) + translation_object = translation_base and logical.startswith(translation_base) and logical != translation['path'] + font_object = font_catalog is not None and logical != font_catalog['path'] + choose = ((args.scope == 'stable' and ((is_snapshot and parts[2] in {'assets', 'catalog', 'text'} and len(parts) >= 4) + or shared_store or translation_object or font_object)) or (args.scope == 'descriptors' and is_snapshot and len(parts) == 3 and parts[2] in {'snapshot.json', 'provenance.json'}) or (args.scope == 'release' and len(parts) >= 3 and parts[:2] == ('releases', args.release_id)) or (args.scope == 'mutable' and logical in args.mutable_path)) @@ -232,6 +374,25 @@ def load_selection(args, *, object_prefix=PREFIX, schemas=('moly-oss-upload/1',) selected = selected[:args.limit] if not selected: raise SafeFailure('no_objects_selected') + if font_catalog is not None: + validate_font_publication(doc) + if resource_catalog is not None: + try: + from resource_publication import validate_upload + validate_upload(doc) + except (OSError, ValueError, KeyError, TypeError) as error: + raise SafeFailure('invalid_resource_catalogue_closure', errorType=type(error).__name__) from None + if translation is not None: + roots = [row for row in doc['files'] if row['path'] == translation['path']] + if len(roots) != 1 or roots[0]['decodedSha256'] != translation['sha256']: + raise SafeFailure('translation_pointer_identity_mismatch') + raw = Path(roots[0]['sourceRaw']).read_bytes() + root = json.loads(raw) + parts = translation['path'].split('/') + if (hashlib.sha256(raw).hexdigest() != translation['sha256'] + or root.get('schema') != 'moly-text-publication/1' or root.get('region') != parts[1] + or root.get('locale') != parts[2] or root.get('revision') != translation['revision']): + raise SafeFailure('local_translation_pointer_changed') # Publishing descriptions requires the actual production descriptor bytes. for row in selected: if row['path'] in {'manifest.json', 'game/manifest.json'} or row['path'].endswith('/snapshot.json'): @@ -291,7 +452,7 @@ def put_one(pool, row, args, stop): # Legacy mutable pointers predate uploader metadata. Verify their # actual stored bytes, bounded to this tiny control-plane surface, # instead of treating absent metadata as proof of their identity. - if previous is None and args.expect_previous_sha256 and key in {PREFIX + name for name in MUTABLE}: + if previous is None and args.expect_previous_sha256 and is_mutable: response = pool.get().get_object(key) try: body = response.read(1048577) diff --git a/tools/deployment/upload_s3.py b/tools/deployment/upload_s3.py index e9e9b24..5730cc4 100644 --- a/tools/deployment/upload_s3.py +++ b/tools/deployment/upload_s3.py @@ -295,11 +295,13 @@ def selection(args): return doc, rows -def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys=None, absent_keys=None): +def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys=None, absent_keys=None, + capacity_bytes=None): """Current encoded object bytes, all bucket prefixes; never decoded lengths. `absent_keys`, when given, receives the selected keys the complete listing does not contain.""" + capacity = CAPACITY if capacity_bytes is None else capacity_bytes incomplete = [] try: if client.get_bucket_versioning(Bucket=target['bucket']).get('Status'): @@ -316,30 +318,40 @@ def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys incomplete.append('multipart_storage_unverified') used = count = 0 listed_keys = set() - continuation = None - for _ in range(10000): - response = client.list_objects_v2(Bucket=target['bucket'], MaxKeys=1000, - **({'ContinuationToken': continuation} if continuation else {})) - if type(response.get('IsTruncated')) is not bool: - raise SafeFailure('bucket_listing_completeness_unverified') - for item in response.get('Contents', []): - size = item.get('Size') - if type(size) is not int or size < 0: - raise SafeFailure('invalid_bucket_object_size') - used += size - count += 1 - key = item.get('Key') - if not isinstance(key, str) or not key: - raise SafeFailure('invalid_bucket_object_key') - listed_keys.add(key) - if not response.get('IsTruncated'): - break - token = response.get('NextContinuationToken') - if not token or token == continuation: - raise SafeFailure('invalid_bucket_listing_cursor') - continuation = token - else: - raise SafeFailure('bucket_listing_page_limit') + # Without an operator capacity budget, only the selected namespaces need + # inventory. Unrelated historic assets do not affect this upload's delta. + prefixes = [None] if capacity else sorted({ + row['objectKey'].rsplit('/', 1)[0] + '/' if '/' in row['objectKey'] else row['objectKey'] + for row in rows + }) + for prefix in prefixes: + continuation = None + seen_tokens = set() + for _ in range(10000): + response = client.list_objects_v2(Bucket=target['bucket'], MaxKeys=1000, + **({'Prefix': prefix} if prefix is not None else {}), + **({'ContinuationToken': continuation} if continuation else {})) + if type(response.get('IsTruncated')) is not bool: + raise SafeFailure('bucket_listing_completeness_unverified') + for item in response.get('Contents', []): + size, key = item.get('Size'), item.get('Key') + if type(size) is not int or size < 0: + raise SafeFailure('invalid_bucket_object_size') + if not isinstance(key, str) or not key: + raise SafeFailure('invalid_bucket_object_key') + if key not in listed_keys: + used += size + count += 1 + listed_keys.add(key) + if not response.get('IsTruncated'): + break + token = response.get('NextContinuationToken') + if not token or token in seen_tokens: + raise SafeFailure('invalid_bucket_listing_cursor') + seen_tokens.add(token) + continuation = token + else: + raise SafeFailure('bucket_listing_page_limit') additional = transfer = existing_count = 0 conflicts = [] conflict_count = 0 @@ -367,13 +379,15 @@ def inspect_capacity(client, target, rows, *, mutable=False, known_existing_keys # reclamation of an old representation during the upload. additional += row['bytes'] transfer += row['bytes'] - return {'capacityBytes': CAPACITY, 'currentObjectBytes': used, 'objects': count, + return {'capacityBytes': capacity or None, 'currentObjectBytes': used if capacity else None, + 'objects': count if capacity else None, 'listedObjectBytes': used, 'listedObjects': count, 'additionalBytesUpperBound': additional, 'transferBytes': transfer, - 'matchingSelectedObjects': existing_count, 'projectedBytesUpperBound': used + additional, + 'matchingSelectedObjects': existing_count, 'projectedBytesUpperBound': used + additional if capacity else None, 'immutableConflictCount': conflict_count, 'immutableConflicts': conflicts, 'complete': not incomplete, 'incompleteReasons': incomplete, - 'withinCapacity': not incomplete and used + additional <= CAPACITY, - 'scope': 'whole-bucket encoded object payload; requires an exclusive publisher; not a quota reservation'} + 'withinCapacity': not incomplete and (not capacity or used + additional <= capacity), + 'scope': ('whole-bucket encoded object payload; requires an exclusive publisher; not a quota reservation' + if capacity else 'selected namespaces; incremental accounting; operator disabled capacity budget')} def previous_digest(client, target, key, existing): @@ -607,7 +621,7 @@ def parse_args(argv=None): parser.add_argument('--profile', help='Optional local AWS SDK profile; otherwise the standard credential chain') parser.add_argument('--scope', choices=['stable', 'release', 'descriptors', 'mutable'], default='stable') parser.add_argument('--release-id') - parser.add_argument('--mutable-path', choices=sorted(common.MUTABLE), action='append', default=[]) + parser.add_argument('--mutable-path', type=common.mutable_path, action='append', default=[]) parser.add_argument('--allow-mutable-update', action='store_true') parser.add_argument('--expect-previous-sha256') parser.add_argument('--limit', type=int) @@ -625,12 +639,16 @@ def parse_args(argv=None): parser.add_argument('--inspect', action='store_true', help='Read-only whole-bucket capacity and object inspection') parser.add_argument('--execute', action='store_true') parser.add_argument('--exclusive-writer', action='store_true', help='Confirm no concurrent publisher can consume the planned bucket capacity') + parser.add_argument('--capacity-bytes', type=int, default=CAPACITY, + help='Operator storage budget in bytes (default: 30 GiB); 0 explicitly disables the budget and inventories selected namespaces') return parser.parse_args(argv) def main(argv=None): global THROTTLE args = parse_args(argv) + if args.capacity_bytes < 0: + raise SafeFailure('capacity_bytes_must_be_nonnegative') if args.max_bytes_per_second is not None: if args.max_bytes_per_second < UPLOAD_CHUNK: raise SafeFailure('max_bytes_per_second_below_one_chunk', minimum=UPLOAD_CHUNK) @@ -646,7 +664,7 @@ def main(argv=None): copy_sources = load_copy_sources(args.copy_source_manifest, args.object_prefix) copyable = [row for row in rows if copy_sources.get(content_identity(row), row['objectKey']) != row['objectKey']] common.emit('selection', destination=target, dryRun=not args.execute, objects=len(rows), - selectedEncodedBytes=sum(row['bytes'] for row in rows), capacityBytes=CAPACITY, + selectedEncodedBytes=sum(row['bytes'] for row in rows), capacityBytes=args.capacity_bytes or None, receipt=str(receipt_path(args, identity)), deleteObjects=False, changeAcl=False, copyCandidates=len(copyable), copyCandidateBytes=sum(row['bytes'] for row in copyable)) if not args.inspect and not args.execute: @@ -674,12 +692,13 @@ def main(argv=None): require_copy_conditions(client) absent_keys = set() plan = inspect_capacity(client, target, rows, mutable=args.scope == 'mutable', - known_existing_keys=known_existing_keys, absent_keys=absent_keys) + known_existing_keys=known_existing_keys, absent_keys=absent_keys, + capacity_bytes=args.capacity_bytes) common.emit('capacity-plan', **plan) if not args.execute: return 0 if not plan['withinCapacity']: - raise SafeFailure('capacity_unverified_or_exceeded', capacityBytes=CAPACITY) + raise SafeFailure('capacity_unverified_or_exceeded', capacityBytes=args.capacity_bytes or None) if plan['immutableConflictCount']: raise SafeFailure('immutable_object_conflicts', count=plan['immutableConflictCount']) return execute(client, target, rows, args, identity, absent_keys, copy_sources) diff --git a/tools/deployment/verified_representations.py b/tools/deployment/verified_representations.py new file mode 100644 index 0000000..3d13871 --- /dev/null +++ b/tools/deployment/verified_representations.py @@ -0,0 +1,147 @@ +"""Reuse a previously observed immutable HTTP body without changing its URL. + +The explicit index is local evidence, never a cloud write permission. Every use +rechecks stored bytes, lossless decoding, addressed identity, and exact headers. +The uploader still independently compares the selected representation to origin. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path, PurePosixPath +import re + +from pack.http_delivery import Representation +from pack.verify import _decode_content + +SCHEMA = 'moly-verified-http-representations/1' +ADDRESS = re.compile(r'(?:asset-store/(?:blobs/[0-9a-f]{2}|packages|catalogs)|catalog-store)/([0-9a-f]{64})\.[a-z0-9]+') +HASH = re.compile(r'[0-9a-f]{64}') +CACHES = frozenset({'public, max-age=31536000, immutable', + 'public, max-age=31536000, immutable, no-transform'}) + + +def digest(body): + return hashlib.sha256(body).hexdigest() + + +def relative(name): + if (not isinstance(name, str) or not name or name.startswith('/') + or any(char in name for char in '\\:%?#') + or any(part in {'', '.', '..'} for part in name.split('/')) + or any(ord(char) < 32 or ord(char) == 127 for char in name)): + raise ValueError('unsafe representation evidence path') + return PurePosixPath(name) + + +@dataclass(frozen=True) +class ReusedRepresentation: + representation: Representation + headers: dict + proof: dict + + def apply(self, row): + # These are precisely the headers recorded at origin, not freshly + # chosen compression/cache policy. Identity means omit Content-Encoding. + row.update(contentType=self.headers['ContentType'], + cacheControl=self.headers['CacheControl'], + representationReuse=self.proof) + + +class VerifiedRepresentations: + def __init__(self, index=None, *, expected_sha256=None): + self.index = None + self.sha256 = None + self.rows = {} + self.used = set() + if index is None: + return + self.index = Path(index).resolve(strict=True) + raw = self.index.read_bytes() + self.sha256 = digest(raw) + if expected_sha256 is not None and self.sha256 != expected_sha256: + raise ValueError('representation index changed') + document = json.loads(raw) + if set(document) != {'schema', 'files'} or document['schema'] != SCHEMA or not isinstance(document['files'], list): + raise ValueError('invalid verified representation index') + for row in document['files']: + if not isinstance(row, dict) or set(row) != {'path', 'objectKey', 'body', 'sha256', 'bytes', 'decodedSha256', 'decodedBytes', 'headers'}: + raise ValueError('invalid verified representation row') + logical = row['path'] + address = ADDRESS.fullmatch(logical) if isinstance(logical, str) else None + if not address or address[1] != row['decodedSha256'] or logical in self.rows: + raise ValueError('representation identity is not a unique addressed object') + relative(logical) + relative(row['objectKey']) + relative(row['body']) + if any(not isinstance(row[key], str) or not HASH.fullmatch(row[key]) for key in ('sha256', 'decodedSha256')): + raise ValueError('invalid representation hash') + if any(type(row[key]) is not int or row[key] < 0 for key in ('bytes', 'decodedBytes')): + raise ValueError('invalid representation length') + headers = row['headers'] + if not isinstance(headers, dict) or set(headers) != {'ContentLength', 'ContentType', 'ContentEncoding', 'CacheControl', 'Metadata'}: + raise ValueError('invalid representation headers') + if (type(headers['ContentLength']) is not int or headers['ContentLength'] != row['bytes'] + or headers['ContentEncoding'] not in {None, 'identity', 'gzip', 'br'} + or headers['CacheControl'] not in CACHES + or not isinstance(headers['ContentType'], str) or not headers['ContentType'] + or any(ord(char) < 32 or ord(char) == 127 for char in headers['ContentType']) + or headers['Metadata'] != {'moly-sha256': row['sha256'], + 'moly-decoded-sha256': row['decodedSha256'], 'moly-decoded-bytes': str(row['decodedBytes'])}): + raise ValueError('representation headers disagree with verified identity') + self.rows[logical] = row + + def select(self, source, logical, object_key, content_type): + row = self.rows.get(logical) + if row is None: + return None + if row['objectKey'] != object_key or row['headers']['ContentType'] != content_type: + raise ValueError('representation URL or media type changed') + source = Path(source).resolve(strict=True) + raw = source.read_bytes() + if len(raw) != row['decodedBytes'] or digest(raw) != row['decodedSha256']: + raise ValueError('representation decoded source changed') + body_path = self.index.parent / relative(row['body']) + # Reject links before resolving, including an intermediate directory. + if body_path.is_symlink() or any(parent.is_symlink() for parent in body_path.parents if parent != self.index.parent): + raise ValueError('representation body must not use symlinks') + body_path = body_path.resolve(strict=True) + if not body_path.is_relative_to(self.index.parent) or not body_path.is_file(): + raise ValueError('representation body escapes evidence directory') + with body_path.open('rb') as stream: + body = stream.read(row['bytes'] + 1) + if len(body) != row['bytes'] or digest(body) != row['sha256']: + raise ValueError('representation stored body changed') + encoding = row['headers']['ContentEncoding'] or 'identity' + decoded = _decode_content(body, 'gzip' if encoding == 'gzip' else 'identity', + 'br' if encoding == 'br' else 'identity', row['decodedBytes']) + if len(decoded) != row['decodedBytes'] or digest(decoded) != row['decodedSha256'] or decoded != raw: + raise ValueError('representation does not decode to its addressed source') + self.used.add(logical) + representation = Representation(body_path, source, encoding, body, len(raw), digest(raw), + (), False, 'verified-immutable-origin') + return ReusedRepresentation(representation, row['headers'], + {'index': str(self.index), 'sha256': self.sha256}) + + def audit(self): + return {'index': None if self.index is None else str(self.index), 'sha256': self.sha256, + 'reused': sorted(self.used), 'unused': sorted(self.rows.keys() - self.used)} + + +def validate_reused_row(row, source, content_type): + """Revalidate reuse evidence immediately before an upload is selected.""" + proof = row.get('representationReuse') + if not isinstance(proof, dict) or set(proof) != {'index', 'sha256'} or not isinstance(proof['sha256'], str) or not HASH.fullmatch(proof['sha256']): + raise ValueError('invalid representation reuse proof') + index = VerifiedRepresentations(proof['index'], expected_sha256=proof['sha256']) + reused = index.select(source, row['path'], row['objectKey'], content_type) + if reused is None: + raise ValueError('representation reuse proof omits object') + expected = reused.representation.manifest_fields + for field in ('sha256', 'bytes', 'decodedSha256', 'decodedBytes', 'contentEncoding', 'deliveryMode', 'requiredAcceptEncoding'): + if row.get(field) != expected[field]: + raise ValueError('representation manifest differs from verified evidence') + if row['cacheControl'] != reused.headers['CacheControl'] or row['contentType'] != reused.headers['ContentType']: + raise ValueError('representation manifest headers differ from verified evidence') + return reused diff --git a/tools/head-portraits/export.mjs b/tools/head-portraits/export.mjs index fca4a59..cbb4976 100644 --- a/tools/head-portraits/export.mjs +++ b/tools/head-portraits/export.mjs @@ -120,7 +120,7 @@ export async function exportHeads(options) { let page; const errors = []; try { - browser = await chromium.launch({ headless: true, ...(options.channel ? { channel: options.channel } : {}) }); + browser = await chromium.launch({ headless: !options.headed, ...(options.channel ? { channel: options.channel } : {}) }); const browserVersion = browser.version(); page = await browser.newPage({ viewport: { width: 1024, height: 900 }, deviceScaleFactor: 1 }); page.setDefaultTimeout(40000); @@ -191,10 +191,10 @@ export async function exportHeads(options) { async function main() { const { values } = parseArgs({ options: { source: { type: 'string' }, out: { type: 'string' }, region: { type: 'string' }, version: { type: 'string' }, - units: { type: 'string' }, channel: { type: 'string' }, python: { type: 'string' }, help: { type: 'boolean' }, + units: { type: 'string' }, headed: { type: 'boolean' }, channel: { type: 'string' }, python: { type: 'string' }, help: { type: 'boolean' }, } }); if (values.help) { - console.log('node tools/head-portraits/export.mjs --source --out --region [--version ] [--units 1,17] [--channel chrome] [--python python]'); + console.log('node tools/head-portraits/export.mjs --source --out --region [--version ] [--units 1,17] [--channel chrome] [--headed] [--python python]'); return; } for (const key of ['source', 'out', 'region']) if (!values[key]) throw new Error(`Missing --${key}`); diff --git a/tools/migrate_canonical_metadata.py b/tools/migrate_canonical_metadata.py index c7831e4..ef68147 100644 --- a/tools/migrate_canonical_metadata.py +++ b/tools/migrate_canonical_metadata.py @@ -25,13 +25,19 @@ # They keep the guard usable in source archives and installed source copies. # Update them only after reviewing the coordinate-writing code. REVIEWED_PRODUCERS = { - 'src/chara/characters.py': 'e18e112c332b31c26110e73aaacfc90911c5124f28782fff8e2da0bcd006a25f', + # Adds source controller/clip identities to the rig only. Removing that + # read-only helper, its call and the one sidecar field reproduces the prior + # reviewed AST; mesh, animation, skinning and GLB writes are unchanged. + 'src/chara/characters.py': '435068f0dd34ff6cb3c329196e0a4165ff78fa409ee93469dec881aff0d7ad70', 'src/chara/motion_library.py': '3ca5702e6ff3d02ee065c32dc5a2e1fe24716999976c135ffa77e52a3b26465c', 'src/chara/player_avatar.py': 'f10f702f6fc3b58f23c1f1a5a388bbe94789e27fec7d93d2b8316abf13a012e1', 'src/chara/avatar_parts.py': '74a42b5519bae8b645850e0281420c707ebac33fa08df3f5bd6326a4dcf89fc3', 'src/chara/avatar_compose.py': '366c5f5e1253728dbbcbb6f476f26dfddaa2062cbe7a6e0ccab5bffd85ec4ca7', 'src/chara/sd_switch.py': '3943cde0837981f94dc9721271313a96dae6a6df6bc928802e2bd82a1f0d473c', - 'src/perf/animations.py': '00f0f58f26895fb5b2163a788ba95a424fd8e0b36052ab2928b77fb1948afe47', + # Only the CLI's default output location changed (MOLY_EXTRACT_OUT); + # reverting that return expression reproduces the previous reviewed hash. + # The coordinate-writing implementation is byte-identical. + 'src/perf/animations.py': '16e31e5b719d47bc04077db95236f3e1efa4295d96fe57433a03f94154e9e851', 'src/core/mesh.py': 'e310116a5d0a905f9cb4f89e2e6c8a5057fd1feb86ded789f00abc4d076fa882', 'src/phenomena/meshes.py': 'd4a20e8f9b0b1d0b9ce7a00ea52ebd63745851c5b9d4b28a6b2d39a46f481a8f', 'src/core/gltf.py': '11129743aab66fa0ed7c6997c7f779f1a06d4cc88a021bfb17a1a42d506451fd',