diff --git a/CHANGELOG.md b/CHANGELOG.md index 29637c44..1a283a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- The image edit pack's `te/manifest.json` (and `edit.json`, which hashes it) is the same on every box: its vision config + is the checkpoint's architecture fields only, not transformers' `to_dict()`, which carried the installed version + (5.20.2 on the reporter's Spark, 5.21.0 on DGX-02: the only difference, with byte-identical weights), so a fresh + `bootstrap.sh` failed `image/packs` on those two files (#84). `prepare.sh` rewrites an existing pack's metadata in + place (no re-pack), the pins are updated, and the twin venv pins `transformers==5.21.0`. - Voice: a long reply is watched for 1.6 s before it starts. In the first second its speech often comes at real time and slows once the chat model writes its next sentences, so the start the rate allowed then still paused in 16 of 84 long replies measured live on DGX-02; with the wait, 3 (each 0.07 to 0.4 s, 35 s or more in). Long replies start diff --git a/docs/dev/IMAGE-EDIT.md b/docs/dev/IMAGE-EDIT.md index a25757cc..e290b6a6 100644 --- a/docs/dev/IMAGE-EDIT.md +++ b/docs/dev/IMAGE-EDIT.md @@ -138,7 +138,9 @@ Status is at the end of this file. ## 6. Pack layout and capability - `te/weights.safetensors` + `vis.*` (bf16; names in `pack_edit.py`); `te/manifest.json` gains - `"vision": {...config, "template_edit": ...}`. + `"vision": {...config, "template_edit": ...}`; `config` holds only the checkpoint's architecture fields + (`pack_edit.VISION_CONFIG`), not transformers' whole `to_dict()`, whose version string made the manifest differ by + box (#84). `tools/spark/prepare.sh` rewrites an older pack's manifest and `edit.json` this way (`pack_edit.normalise`). - `vae/weights.safetensors` + `vae.encoder.*`, `vae.quant_conv.*` (all 128 moment channels; the twin selects the first 64 outputs), `vae.enc_latents_*`; `vae/manifest.json` gains an `encoder` member with its own convs / norms / downs maps; top-level decoder maps stay separate. - `/edit.json`: `{"format": "stk-edit/1", "refs_max": 5, "manifest_sha256": {"te": ..., "vae": ...}}`, written by the pack step only when both parts diff --git a/tools/spark/prepare.sh b/tools/spark/prepare.sh index d660a34b..36204779 100755 --- a/tools/spark/prepare.sh +++ b/tools/spark/prepare.sh @@ -103,6 +103,9 @@ if packs; then [[ -f $W/vae/manifest.json ]] || { step "pack vae"; $T pack --model $MODEL --precision vae --out $W/vae; } # image edit: the vision tower into te/, the VAE encoder into vae/, then edit.json (the tool lists image_edit only with it) [[ -f $W/edit.json ]] || { step "pack edit"; $T pack --model $MODEL --precision edit --out $W; } +# an edit pack from before #84: its te/manifest.json carried the installed transformers' version (so its pin differed by +# box); rewritten with the checkpoint's vision fields only, and edit.json with it (metadata, no re-pack) +python -c "import sys; sys.path.insert(0, '$TWIN'); from stk_twin.pack_edit import normalise; normalise('$W') and print('[prepare] edit pack metadata normalised (#84)')" python -c "import sys; sys.path.insert(0, '$TWIN'); from stk_twin.files import readable; readable('$W')" fi step "image done$(packs || echo ' (DOWNLOAD_ONLY: the checkpoints only)'): $(du -sh $W | cut -f1) in $W" diff --git a/tools/twin/packs/built/qwen-image-2.1-turbo.json b/tools/twin/packs/built/qwen-image-2.1-turbo.json index b9fdffe0..26c0450b 100644 --- a/tools/twin/packs/built/qwen-image-2.1-turbo.json +++ b/tools/twin/packs/built/qwen-image-2.1-turbo.json @@ -1,10 +1,10 @@ { "repo_id": "Qwen/Qwen-Image-2.1-Turbo", "revision": "d65dbc9a7e8f6b5479e33dee6030eaab2a906509", - "note": "what tools/spark/prepare.sh builds into weights/qwen-image-2.1 for the qwen-image-2.1-turbo tool (IMAGE_MODELS=qwen-image-2.1-turbo, PRECISIONS=fp8s): the Turbo DiT pack and the shared text-encoder and VAE packs, from the pinned sources (packs/sources/); built on DGX-01 (GB10) 2026-10-21; te/, vae/ and edit.json since the edit pack (prepare.sh's `pack edit`, PR #78: the vision tower into te/, the VAE encoder into vae/), pinned from the DGX-02 build of 2026-10-27, whose te/ and vae/ without the edit step matched the 2026-10-21 pins byte for byte. The per-GPU Triton cubins (turbo-fp8s/triton/) are compiled on the box and not listed. Checked by tools/package/verify_assets.py", + "note": "what tools/spark/prepare.sh builds into weights/qwen-image-2.1 for the qwen-image-2.1-turbo tool (IMAGE_MODELS=qwen-image-2.1-turbo, PRECISIONS=fp8s): the Turbo DiT pack and the shared text-encoder and VAE packs, from the pinned sources (packs/sources/); built on DGX-01 (GB10) 2026-10-21; te/, vae/ and edit.json since the edit pack (prepare.sh's `pack edit`, PR #78: the vision tower into te/, the VAE encoder into vae/), pinned from the DGX-02 build of 2026-10-27, whose te/ and vae/ without the edit step matched the 2026-10-21 pins byte for byte. te/manifest.json carries the vision tower's checkpoint fields only (not the installed transformers' version, which made it and edit.json differ by box, #84). The per-GPU Triton cubins (turbo-fp8s/triton/) are compiled on the box and not listed. Checked by tools/package/verify_assets.py", "selected_files": [ - {"path": "edit.json", "size_bytes": 221, "sha256": "d426ea093e524b5e90337c74d2b66a0db865f49c658dbf7cfed30cbe3ae20530"}, - {"path": "te/manifest.json", "size_bytes": 131480, "sha256": "de7a0a752d7ac0fd83dd9d158994e0658bdcf24992788415369c4763c250a95a"}, + {"path": "edit.json", "size_bytes": 221, "sha256": "64638c30c878558e960c98d8a2de4c195dfb77628d589a2418c8605773f9683f"}, + {"path": "te/manifest.json", "size_bytes": 130922, "sha256": "91fd46ff3a3054fa5d08e50d0395b17c4ca74967edc7f6f88290708cb6576cfa"}, {"path": "te/tokenizer.json", "size_bytes": 11422650, "sha256": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506"}, {"path": "te/tokenizer/chat_template.jinja", "size_bytes": 5292, "sha256": "3636d0f0bd6bef02654cdffdc447b79cb2cef8ab02cc75267345946291a489e4"}, {"path": "te/tokenizer/tokenizer.json", "size_bytes": 11422650, "sha256": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506"}, diff --git a/tools/twin/pod/setup.sh b/tools/twin/pod/setup.sh index 09d575fc..12f59086 100755 --- a/tools/twin/pod/setup.sh +++ b/tools/twin/pod/setup.sh @@ -6,9 +6,10 @@ mkdir -p $STK/src $STK/models if [[ ! -f $STK/venv/bin/activate ]]; then python -m venv --system-site-packages $STK/venv source $STK/venv/bin/activate - # the image's torch is a pre-release pip would replace: pin it, triton and numpy to what is installed + # the image's torch is a pre-release pip would replace: pin it, triton and numpy to what is installed. transformers + # exactly: the packs are built through it (the image pins were made with 5.21.0, #84) pip freeze --all | grep -iE '^(torch|triton|numpy)==' > $STK/venv/constraints.txt - pip install -q -c $STK/venv/constraints.txt "diffusers==0.41.0" "transformers>=4.57" accelerate + pip install -q -c $STK/venv/constraints.txt "diffusers==0.41.0" "transformers==5.21.0" accelerate fi [[ -d $STK/src/TensorFold ]] || git clone -q --depth 1 --branch $TENSORFOLD_TAG https://github.com/ashhart/TensorFold $STK/src/TensorFold pip install -q --no-deps -e $STK/src/TensorFold diff --git a/tools/twin/stk_twin/pack_edit.py b/tools/twin/stk_twin/pack_edit.py index 05f5e495..d5cc0ab3 100644 --- a/tools/twin/stk_twin/pack_edit.py +++ b/tools/twin/stk_twin/pack_edit.py @@ -15,6 +15,14 @@ from .pack import _save from .files import readable +# The vision tower's architecture as the checkpoint's config.json gives it, in this order: the fields the engine reads +# (src/engines/qwen_image/vision.zig) and the shapes they set. Not the whole `to_dict()`: that also carries the +# installed transformers' version and its defaults, so te/manifest.json (and edit.json, which hashes it) differed +# between boxes with byte-identical weights (#84: transformers 5.20.2 against 5.21.0). +VISION_CONFIG = ("depth", "hidden_size", "hidden_act", "intermediate_size", "num_heads", "in_channels", "patch_size", + "spatial_merge_size", "temporal_patch_size", "out_hidden_size", "num_position_embeddings", + "deepstack_visual_indexes") + def base(path: Path, model: str): man = json.loads((path / "manifest.json").read_text()) @@ -43,7 +51,7 @@ def write_vision(pipe, root: Path): # Conv3d [D,3,2,16,16] on one patch is the twin's GEMM [D,1536]. No rank-5 pack tensor. value = value.reshape(value.shape[0], -1) tensors[key] = value.contiguous() - cfg = visual.config.to_dict() + cfg = {k: visual.config.to_dict()[k] for k in VISION_CONFIG} if cfg["hidden_size"] // cfg["num_heads"] != 72 or cfg["spatial_merge_size"] != 2: raise ValueError("vision pack requires 72-wide heads and merge size 2") man["vision"] = {"config": cfg, "template_edit": TEMPLATE, "heads_padded": 128, @@ -112,10 +120,36 @@ def write(pipe, out: str | Path): marker = root / "edit.json" marker.unlink(missing_ok=True) counts = {"te": write_vision(pipe, root), "vae": write_encoder(pipe, root)} - hashes = {name: hashlib.sha256((root / name / "manifest.json").read_bytes()).hexdigest() for name in counts} + return {"pack": str(root), "precision": "edit", "tensors": counts, **mark(root)} + + +def mark(root: Path) -> dict: + """edit.json for the te/ and vae/ manifests now in ROOT.""" + hashes = {name: hashlib.sha256((root / name / "manifest.json").read_bytes()).hexdigest() for name in ("te", "vae")} result = {"format": "stk-edit/1", "refs_max": 5, "manifest_sha256": hashes} temp = root / "edit.json.tmp" temp.write_text(json.dumps(result, indent=1)) - temp.replace(marker) + temp.replace(root / "edit.json") readable(root) - return {"pack": str(root), "precision": "edit", "tensors": counts, **result} + return result + + +def normalise(out: str | Path) -> bool: + """An edit pack written before VISION_CONFIG (#84): te/manifest.json rewritten with only those fields, as `write` + now writes it (the rest of the file is kept byte for byte: json round-trips it), then edit.json for it. Metadata + only; the weights are not touched. True when it rewrote anything.""" + root = Path(out) + te, marker = root / "te" / "manifest.json", root / "edit.json" + if not marker.exists(): + return False + man = json.loads(te.read_text()) + cfg = man["vision"]["config"] + if list(cfg) == list(VISION_CONFIG): + return False + man["vision"]["config"] = {k: cfg[k] for k in VISION_CONFIG} + marker.unlink() # no marker naming a manifest that is being replaced: edits stay off until both are written + temp = te.with_name("manifest.json.tmp") + temp.write_text(json.dumps(man, indent=1)) + temp.replace(te) + mark(root) + return True