From f745e7abc20890e0d25f37b7db24c37902dc6888 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 7 Oct 2026 19:54:51 +0200 Subject: [PATCH 01/21] docs(spec): APR-EMBED-001 EmbeddingGemma 2 support, verbatim; mint EG-0..EG-9 The operator's spec, committed byte-for-byte (sha256 7f825e1f1f6a3215...). Epic #4898 on 0.72.0; rows EG-0..EG-9 are #4899..#4908, minted with `pmat work add --epic 4898 --github-issue N`. Refs #4898 #4899 Co-Authored-By: Claude Opus 5.5 (1M context) Agent: apr-embed --- docs/roadmaps/entries/PMAT-4899.yaml | 18 ++ docs/roadmaps/entries/PMAT-4900.yaml | 18 ++ docs/roadmaps/entries/PMAT-4901.yaml | 18 ++ docs/roadmaps/entries/PMAT-4902.yaml | 18 ++ docs/roadmaps/entries/PMAT-4903.yaml | 18 ++ docs/roadmaps/entries/PMAT-4904.yaml | 18 ++ docs/roadmaps/entries/PMAT-4905.yaml | 18 ++ docs/roadmaps/entries/PMAT-4906.yaml | 18 ++ docs/roadmaps/entries/PMAT-4907.yaml | 18 ++ docs/roadmaps/entries/PMAT-4908.yaml | 18 ++ .../APR-EMBED-001-embeddinggemma-2-support.md | 294 ++++++++++++++++++ 11 files changed, 474 insertions(+) create mode 100644 docs/roadmaps/entries/PMAT-4899.yaml create mode 100644 docs/roadmaps/entries/PMAT-4900.yaml create mode 100644 docs/roadmaps/entries/PMAT-4901.yaml create mode 100644 docs/roadmaps/entries/PMAT-4902.yaml create mode 100644 docs/roadmaps/entries/PMAT-4903.yaml create mode 100644 docs/roadmaps/entries/PMAT-4904.yaml create mode 100644 docs/roadmaps/entries/PMAT-4905.yaml create mode 100644 docs/roadmaps/entries/PMAT-4906.yaml create mode 100644 docs/roadmaps/entries/PMAT-4907.yaml create mode 100644 docs/roadmaps/entries/PMAT-4908.yaml create mode 100644 docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md diff --git a/docs/roadmaps/entries/PMAT-4899.yaml b/docs/roadmaps/entries/PMAT-4899.yaml new file mode 100644 index 0000000000..0c7e96e0f3 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4899.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4899 + github_issue: 4899 + item_type: task + title: 'EG-0: Measure — probe, oracle, contracts, corpus (no engine code)' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:53:52Z + updated: 2026-10-07T17:53:52Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:measurement + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4900.yaml b/docs/roadmaps/entries/PMAT-4900.yaml new file mode 100644 index 0000000000..f21517f2ce --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4900.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4900 + github_issue: 4900 + item_type: task + title: 'EG-1: Load — GGUF, tokenizer, kind' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:53:58Z + updated: 2026-10-07T17:53:58Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4901.yaml b/docs/roadmaps/entries/PMAT-4901.yaml new file mode 100644 index 0000000000..dc0d9b3542 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4901.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4901 + github_issue: 4901 + item_type: task + title: 'EG-2: Embed on CPU' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:04Z + updated: 2026-10-07T17:54:04Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4902.yaml b/docs/roadmaps/entries/PMAT-4902.yaml new file mode 100644 index 0000000000..603b394cd1 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4902.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4902 + github_issue: 4902 + item_type: task + title: 'EG-3: Serve' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:09Z + updated: 2026-10-07T17:54:09Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4903.yaml b/docs/roadmaps/entries/PMAT-4903.yaml new file mode 100644 index 0000000000..601f9200be --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4903.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4903 + github_issue: 4903 + item_type: task + title: 'EG-4: CUDA on sm_89 and sm_121' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:14Z + updated: 2026-10-07T17:54:14Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4904.yaml b/docs/roadmaps/entries/PMAT-4904.yaml new file mode 100644 index 0000000000..39f9079eb5 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4904.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4904 + github_issue: 4904 + item_type: task + title: 'EG-5: Refusals, apr qa, the ladder' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:19Z + updated: 2026-10-07T17:54:19Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4905.yaml b/docs/roadmaps/entries/PMAT-4905.yaml new file mode 100644 index 0000000000..fc0263d8dd --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4905.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4905 + github_issue: 4905 + item_type: task + title: 'EG-6: Model ops round trip' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:23Z + updated: 2026-10-07T17:54:23Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4906.yaml b/docs/roadmaps/entries/PMAT-4906.yaml new file mode 100644 index 0000000000..475b22d153 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4906.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4906 + github_issue: 4906 + item_type: task + title: 'EG-7: Fine-tune' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:28Z + updated: 2026-10-07T17:54:28Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4907.yaml b/docs/roadmaps/entries/PMAT-4907.yaml new file mode 100644 index 0000000000..db21b1ff4a --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4907.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4907 + github_issue: 4907 + item_type: task + title: 'EG-8: Prune, then distill' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:33Z + updated: 2026-10-07T17:54:33Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/roadmaps/entries/PMAT-4908.yaml b/docs/roadmaps/entries/PMAT-4908.yaml new file mode 100644 index 0000000000..b9211048e9 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4908.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4908 + github_issue: 4908 + item_type: task + title: 'EG-9: Evaluate, measure, dogfood' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:38Z + updated: 2026-10-07T17:54:38Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 diff --git a/docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md b/docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md new file mode 100644 index 0000000000..22ca3d9e23 --- /dev/null +++ b/docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md @@ -0,0 +1,294 @@ +# APR-EMBED-001 — EmbeddingGemma 2 on every `apr` verb: embed, serve, model ops, fine-tune, prune, distill + +Status: DRAFT v0.1 for quorum (one agy · one claude · one apr) · 2026-10-07 · owner: Noah Gift · train 0.72 `[O 2026-10-07]` +Drop at `docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md`. Run: `Implement docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md autonomously.` +Marks: `[V]` measured 2026-10-07 on `origin/main` @ `409800713` by the probe beside it (grep and a YAML parse in a read-only clone; EG-0 re-binds code seams with `pmat query`) · `[C]` computed, arithmetic shown · `[O]` operator statement · `[A]` taken from a spec, or an estimate · `[X]` third party, never a claim · `[U]` unmeasured; the row that measures it is named. + +--- + +## ELI5 + +**Goal.** `apr` runs Google's EmbeddingGemma 2 from the Unsloth GGUF. Every verb either works with a receipt or says no by name. + +**How it works.** +- A model gets a *kind*: generative or embedding. The kind is a type. It is read from the file. +- Every verb in the registry chooses, per kind: `receipt`, `refuse`, or `no-model`. +- Then the training verbs: fine-tune, prune, distill. They are judged by retrieval on held-out data. + +**What is true today.** +- Nothing loads it. The architecture string has zero hits in the code. +- The pinned llama.cpp oracle is three weeks older than the model. +- If the file merely loaded, the embeddings route would return a wrong vector with HTTP 200. + +**What stops the line.** A wrong vector that looks right · a dropped image · float16 math · a pin change without the operator's yes. + +**Done when.** 114 of 114 verbs classified · 6 of 6 files at parity on both required GPU hosts · 0 silent-wrong cells · one fine-tune that beats its base on held-out data, with controls. + +| Question | Answer | +|---|---| +| Does `apr` load `gemma-embedding2` today? | No `[V]` | +| Does Unsloth prune or distill this model? | No `[X]` — neither word is in its README or its embedding guide | +| Does the Unsloth GGUF hold the vision and audio encoders? | No `[C]` — 558 MB at 16 bits is the text tower | +| Can upstream llama.cpp serve image or audio embeddings? | No `[X]` — its issue 30082 was closed as not planned | +| Is 0.72 "Train What You Serve" on `main`? | No `[V]` — `docs/roadmaps/epics.yaml` maps 0.72 to Agent Ready | + +## Purpose and terms posture + +- The goal is an open-source Rust ML framework. Models are not sold. Nothing here is built to compete with any provider. +- No hosted-model output is ever a training target. Training data here is gold labels and public pair data with a recorded licence. +- Claude Code is used only by its paying account owner. No shared credentials. No free accounts. +- Public wording never positions a model as a replacement for a commercial service. +- EmbeddingGemma 2 is Apache-2.0 and its card binds deployments to the Gemma Prohibited Use Policy `[X]`. Any weights derived here (fine-tuned, pruned, distilled, re-quantised) keep that licence and policy reference in their stamp. **This spec publishes no crate and no weights.** + +--- + +## §0 Operating assumptions + +1. **"0.7.2" is train 0.72.** Tag `v0.70.2` exists (`89e261cda`, 2026-10-07T12:33Z) `[V]`. +2. **Guide coverage.** `sovereign-ai-stack-claude-project-guide.md` (snapshot 2026-07-02) does not cover EmbeddingGemma, Unsloth, model kinds, or the 0.7x trains. Only the publish gate and the workflow rules below are guide-grounded. +3. **Two artifacts.** The GGUF files are the inference artifact. The safetensors checkpoint is the training source. Training verbs take `.apr`. +4. **Kind is a type.** `ModelKind::{Generative, Embedding}`. Verb dispatch is an exhaustive `match`, so a new kind cannot compile until every verb decides. +5. **Refusal is a shipping behaviour, never a success condition.** `refuse` cells are counted on their own line and never as green. +6. **Measure, then build.** EG-0 lands with no engine code. +7. **One declaration per fact.** The registry and the capability contract are extended. No parallel table is minted. +8. **Cross-repo work is filed, not done.** Oracle build and competitor leg are forjar resources in paiml/infra. Cells that need them are `NotRun{NoDeclaredExecutor}` until they land. +9. **The operator has no pre-steps.** Anything only Noah can do is a §7 STOP. +10. **The two attachments of 2026-10-07** (Jev-Omni, LLM2Vec) are read as adjacent scope. They are Appendix B, not 0.72 rows. + +--- + +## §1 Ground truth (baseline, frozen; EG-0 re-derives it; never quote as current) + +### 1.1 The tree + +| # | Fact | Mark | Source / probe | +|---|---|---|---| +| G1 | The verb registry holds **114** commands in 13 categories. **42** take a model, **72** do not | `[V]` | parse `commands:` in `contracts/apr-cli-commands-v1.yaml`, count `requires_model` | +| G2 | `gemma-embedding` in any spelling: **0** hits in `crates/` and `contracts/`. `gemma4`: **0** hits. Architecture literals: `gemma` 119, `gemma2` 51, `gemma3` 24, `gemma3n` 5 | `[V]` | zero hits: `grep -rIilE` over both trees; literals: `grep -rhoE '"gemma[a-z0-9_-]*"' crates --include='*.rs' \| sort \| uniq -c` | +| G3 | The contract gate treats any name starting with `gemma` as the Gemma family and fails loud unless it is exactly v1 or v2. So today's expected behaviour is a loud refusal `[C]`. The gate's own doc states that alternating local and global attention is **not implemented** | `[V]` | `is_gemma_family`, `is_gemma1_supported`, `is_gemma2_supported` in `crates/aprender-serve/src/contract_gate.rs`; `is_gemma1`, `is_gemma2` in `crates/aprender-serve/src/gguf/config.rs` | +| G4 | `apr embed` is a BERT bi-encoder: WordPiece, `.apr` only, layer sizes typed as flags, forward in `aprender::models::bert` and not in realizar | `[V]` | `crates/apr-cli/src/commands/embed.rs` | +| G5 | `/v1/embeddings` mean-pools the final hidden state of whatever model is loaded, L2-normalises it, and returns the trunk width. The contract pins "dim == model hidden size" | `[V]` | `crates/aprender-serve/src/api/realize_handlers_embed_completion.rs`; EMBEDDINGS-MODEL-BACKED in `contracts/apr-serve-openai-compat-v1.yaml` | +| G6 | The capability registry derives `CausalMask` for **every** architecture (`op_derivation.always`) and maps a gated MLP to the `SwiGLU` op. `GeluMlp`, `LayerNorm`, `AttnFinalSoftcap`, `PostAttnFfnNorm` are GPU-unsupported. `BF16`, `F16`, `Q8_0`, `Q4_K`, `Q5_K`, `Q6_K` are GPU-supported | `[V]` | `contracts/apr-model-capability-v1.yaml` (`ops`, `op_derivation`, `quant_types`) | +| G7 | An LLM-as-embedder path through realizar already exists, with query and passage prefixes. The other real embedder is fastembed, an optional external dependency on ONNX Runtime | `[V]` | `crates/aprender-rag/src/embed/nemotron.rs`; `FastEmbedder` in `crates/aprender-rag/src/embed/mod.rs`; `crates/aprender-rag/Cargo.toml` | +| G8 | `finetune` (auto, full, LoRA, QLoRA), `prune` (magnitude, structured, depth) and `distill` (teacher to student, YAML config) all take `.apr` | `[V]` | module docs of `crates/apr-cli/src/commands/finetune.rs`, `prune.rs`, `distill.rs` | +| G9 | Contrastive seams exist: `info_nce_loss`, `InfoNCELoss`, `TripletLoss`; `ContrastiveTask`; `SimCSE`; a pair-data crate. Masked next-token prediction, Matryoshka and multiple-negatives ranking: **0** hits each | `[V]` | `crates/aprender-core/src/loss/loss.rs`; `crates/aprender-core/src/nn/self_supervised.rs`; `crates/aprender-core/src/nn/self_supervised_byol_simcse.rs`; `crates/aprender-contrastive-data` | +| G10 | Oracle pin: `build_commit = "d1d3c3396"`, `pinned_on = "2026-09-15"`, `pin_expiry = "2026-12-01"`. Upstream support for this model merged 2026-10-06 `[X]`. So the pinned oracle cannot load it `[C]` | `[V]` | `scripts/llama_pin.toml`; resolver `scripts/llama_bin.sh` | +| G11 | Parity floor `min_cosine: 0.98` has a measured basis on autoregressive models over at least 64 positions. A model listed without its own `min_cosine` is fail-closed | `[V]` | `evidence/parity/thresholds.yaml` | +| G12 | Ladder: required hosts lambda (sm_89) and gx10 (sm_121). `cells.verbs` is `run, chat, serve, code`. All 8 rungs are Qwen. The inventory sweep measures every matching file a host holds. A Q4_K rung must be required and must claim cuda | `[V]` | `contracts/model-capability-ladder-v1.yaml` (`hosts`, `cells`, `inventory`, the 3712 comment) | +| G13 | Train map: 0.72 is E5 "Agent Ready" (issue 4000, budget 5, 9 rows) and its plan puts new model support out of scope. 0.74 is "Any Model". 0.75 is "CRUX Fine-Tune/Distill". "Train What You Serve": **0** hits | `[V]` | `docs/roadmaps/epics.yaml`; `docs/specifications/EPIC-0.72-agent-ready-plan.md` §5 | +| G14 | A tag carries named scope only through the release-critical table | `[V]` | `docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md` §1.5 | +| G15 | Debt baselines this spec must not raise: shared attention and FFN blocks are called by 0 of 8 production forward files; 42 files match on quant type | `[A]` 2026-09-23 | `docs/specifications/EPIC-0.74-any-model-plan.md` §1 and §6 | +| G16 | No fleet host declares Unsloth. `apr` concedes in-loop QLoRA throughput on GPU | `[A]` 2026-09-23 | `docs/specifications/EPIC-0.75-crux-finetune-distill-plan.md` §1 | +| G17 | Capacity sample: 8 to 20 merged PRs per train | `[A]` 2026-09-10 | `docs/specifications/06x-release-schedule.md` §1.2 | + +### 1.2 The model — all `[X]` until EG-0 measures the files + +| # | Fact | Source | +|---|---|---| +| M1 | Architecture `gemma-embedding2`. 24 layers, width 512, FFN hidden 2,048, 4 heads, KV heads 2 (local) and 1 (global), local to global 5:1, sliding window 1,024, vocabulary 262,144, gated FFN with GELU, mean pooling, projection 512 to 768, context 8,192 | model card | +| M2 | The text encoder is bidirectional | Unsloth PR 12865 | +| M3 | Unsloth files: UD-Q4_K_XL 176 MB, UD-Q5_K_XL 210 MB, UD-Q6_K_XL 249 MB, Q8_0 310 MB, BF16 558 MB, F16 558 MB. Listed size 0.3B parameters. 558 MB ÷ 2 bytes ≈ 279 M parameters: the text tower alone `[C]` | Unsloth GGUF page | +| M4 | Output dimensions 768, 512, 256, 128. A truncated vector must be re-normalised | model card | +| M5 | Seven task prompts (`SearchQuery`, `QuestionAnswering`, `FactChecking`, `CodeRetrieval`, `Classification`, `Clustering`, `SentenceSimilarity`) and the document form `title: {title or none} \| text: {content}`. Omitting the prompt lowers quality without an error | model card | +| M6 | Float16 activations return NaN or silently degraded vectors. bfloat16 and float32 are safe. Unsloth pools in float32 for this reason | model card; Unsloth PR 12865 | +| M7 | Upstream llama.cpp: text embeddings work. `/embedding` with `image_data` answers 200 and ignores the image; three different images gave identical vectors | llama.cpp PR 30054, issue 30082 | +| M8 | What Unsloth does with it: text serving through llama.cpp (`--embeddings --pooling mean`); LoRA, QLoRA and full fine-tuning (rank 16, alpha 32, targets q/k/v/o/gate/up/down, multiple-negatives ranking loss, bf16 on, fp16 off); save adapters, save merged, push. Vision and audio layers are excluded from LoRA by default | Unsloth model guide, embedding guide, PR 12865 | +| M9 | `[U]` → EG-0: per-tensor quant types inside the three UD files · exact names of five of the six files · metadata keys (causal flag, pooling type, window pattern, per-layer KV heads) · whether the 512 to 768 projection is in the main GGUF · whether the trunk uses post-norms, QK-norm or RoPE · whether F16 weights degrade on the oracle itself | — | + +**Consequence of G5 + M1 `[C]`.** The trunk width (512) is also a legal truncated dimension. A build that loads this file and skips the projection returns a 512-long, unit-norm, plausible vector. A length check cannot see it. Only a value check can. + +--- + +## §2 Invariants (each has a falsifier in §9) + +| # | Invariant | +|---|---| +| **I-1** | **Publishing: clean-room CI on the tag is the hard gate, named first.** No workflow runs `cargo publish`. Never `--allow-dirty`. No EG row publishes a crate or weights; `apr publish` is exercised with `--dry-run` only | +| **I-2** | **Totality.** Every registry command declares one verdict per model kind: `receipt`, `refuse`, or `no-model`. There is no fourth state | +| **I-3** | **No silent wrong answer.** A `receipt` cell that exits 0 has passed its check. A `refuse` cell exits non-zero and names the verb to use. A permanent refusal is `Refused{NotThisKind}`; an owed one is `Refused{NotYet, removed_by: }` | +| **I-4** | **The output is the projected vector.** The default response is 768 long. A truncated response equals the leading dimensions of the 768 vector, re-normalised. It is never the pooled trunk state | +| **I-5** | **Finite or refuse.** No non-finite value leaves `apr`. Activations for this architecture never run in float16; a float16 kernel dispatch refuses by name | +| **I-6** | **Batch and order invariance.** A text embeds the same alone, in a batch, and at any batch position | +| **I-7** | **Media is refused, never dropped.** An image, audio or video input to a file with no such encoder gets a 4xx that names the missing encoder | +| **I-8** | **Parity is against an independent implementation on the same file.** The oracle is llama.cpp at the declared pin. The threshold is set by the thresholds file's own method: known-good and known-bad pairs, n = 5, each required host. GPU against CPU the same way | +| **I-9** | **Per-layer facts come from the file.** Attention type, KV-head count, pooling and the causal flag are read from metadata and tensor shapes. Never from layer-index arithmetic. Never from an architecture-level constant | +| **I-10** | **No new debt.** The new forward path calls the shared attention and FFN blocks. No quant-type `match` is added outside the type table | +| **I-11** | **Trained weights are judged by retrieval, with controls.** A fine-tune must beat its base beyond the 3-seed band. A shuffled-pair run must not | +| **I-12** | **Receipts carry identity.** `apr` version and sha, feature set, model sha256, quant, backend, host, oracle build, task prompt, dimension. A receipt whose oracle build differs from the pin on `main` at the cut is STALE | + +--- + +## §3 Rows (EV order; one ticket per session; one parent issue per PR) + +Selector: each session takes the **first row whose `done_when` fails at `origin/main`**, does that row only, and stops. +Until a `receipt` cell is green it refuses by name with `removed_by`. The exit bar counts greens only. + +| Row | Title | Verbs it receipts | Deliver | done_when | K̂ `[A]` | +|---|---|---|---|---|---| +| **EG-0** | Measure: probe, oracle, contracts, corpus. **No engine code** | — | (a) Probe receipt: all 42 model-taking verbs and the 8 model-adjacent ones on the UD-Q4_K_XL file, exit code and first line each. (b) `apr inspect` and `apr tensors` on 6 of 6 files: resolves M9. (c) Oracle: the pinned build refuses the file; a build at the upstream merge of PR 30054 or later writes golden vectors and token ids for the corpus. (d) Registry gains a per-command, per-kind verdict; the totality guard runs in `ci / gate`. (e) Capability contract gains the new ops and makes `CausalMask` kind-conditional, source and mirror together. (f) Forward contract (local and global attention, mean pool, projection, normalise, truncation) passes `pv validate` with no anonymous obligation. (g) Pre-registered corpus with a sha256 manifest: texts × 7 prompts and the document form; lengths 8, 128, 1,024, 2,000 and 8,000 tokens; a batch-invariance set; a retrieval micro-set; a held-out fine-tune set | all on `main`; the guard has a first green and its mutation RED in the required check; every M9 item has a value and its command | 80 | +| **EG-1** | Load: GGUF, tokenizer, kind | `pull` `list` `rm` `tokenize` `capability` `inspect` `debug` `validate` `lint` `tensors` `diff` `hex` `tree` `flow` | Loader admits the architecture as `ModelKind::Embedding` by a distinct path; the `gemma` prefix match is not widened. Per-layer KV heads and attention type read from the file (I-9). SentencePiece vocabulary of 262,144. Model-family contract for the variant | token ids equal the oracle's on 100% of the corpus; the 14 verbs green on 6 of 6 files | 100 | +| **EG-2** | Embed on CPU | `embed` `trace` | Forward in realizar on the shared blocks (I-10): local and global attention, gated GELU FFN, mean pool over real tokens, projection, L2 normalise, truncation. `apr embed` takes a GGUF and reads sizes from the file; `--task`, `--title`, `--dim`. The BERT path is unchanged | parity (I-8) on 6 of 6 files × 7 prompts × 4 dimensions × 5 lengths; I-4 and I-6 green; BERT `apr embed` tests still green | 150 | +| **EG-3** | Serve | `serve` | Kind-aware routes: `/v1/embeddings`, `/api/embeddings`, `/realize/embed` use the EG-2 path; generative routes answer 4xx naming the kind. The request's `dimensions` field maps to truncation. Media refused (I-7). The contract obligation becomes "dim == hidden size for the generative kind; dim == declared output for the embedding kind" | HTTP and CLI vectors equal for the same input; I-7 green; the amended contract passes `pv validate` | 80 | +| **EG-4** | CUDA on sm_89 and sm_121 | `parity` `profile` `ptx-map` | GPU forward for every op EG-0 found, with the GELU gate (not the SiLU one). Float16 poka-yoke (I-5). Long-input cells at 2,000 and 8,000 tokens | GPU-against-CPU parity on both required hosts, 6 of 6 files; I-5 green; a line that reads `falling back to CPU` is RED | 120 | +| **EG-5** | Refusals, `apr qa`, the ladder | `qa` and the five refusals: `run` `chat` `code` `rerank` `showcase` | Named refusals (I-3). `apr qa` gains a golden-embedding gate for this kind. The ladder contract gains kind-aware cells (`embedding: [embed, serve]`); the inventory sweep cannot judge an embedding file on generative verbs. One rung per file with its sha256 | `scripts/check_model_ladder.sh` green on both hosts with the new rungs; 5 of 5 refusals by name | 80 | +| **EG-6** | Model ops round trip | `import` `convert` `export` `rosetta` `stamp` `compile` `quantize` `merge` `shard` `unshard` `encrypt` `decrypt` `publish` (dry run) | gguf ↔ safetensors ↔ `.apr`. Each output is re-embedded and compared with its source by I-8. An exported GGUF must load in the oracle. `quantize` from BF16 to Q8_0 and Q4_K is compared with Unsloth's own files, report only | 13 verbs green; round-trip parity on the corpus; the oracle loads every exported GGUF | 120 | +| **EG-7** | Fine-tune | `finetune` `tune` `diagnose` | Backward through the encoder. In-batch-negative contrastive loss on pairs and triplets. A batch sampler with no duplicates. bf16 or f32 only. Method order: LoRA, then QLoRA, then full; each is its own cell. Adapter and merged outputs. FILE ONLY: an Unsloth leg in paiml/infra, pinned in forjar | I-11 green over 3 seeds; the merged model passes EG-2 and EG-6 checks; no non-finite loss | 180 | +| **EG-8** | Prune, then distill | `prune` `distill` | Depth pruning in whole local-to-global periods of 6 layers, unless the target format carries a per-layer pattern (I-9). Magnitude and structured pruning report curves. `apr distill` from the original to the pruned student on an embedding loss | a quality-retention curve published whichever way it falls; no threshold until 3 records; every output finite and loadable | 150 | +| **EG-9** | Evaluate, measure, dogfood | `bench` `eval` `check` `qualify` `compare-hf` `rag` | Throughput is report only: texts and tokens per second at batch 1, 8 and 32, both hosts, beside the oracle server on the same file, under the exclusive GPU lock. An `Embedder` for this model beside `NemotronEmbedder`. `apr rag index` and `apr rag query` on one real corpus. FILE ONLY: the semantic leg of the infra RAG eval | one retrieval receipt beside the `FastEmbedder` baseline; APR-OBS timing rows present for `embed` and `serve` | 120 | + +**Cell arithmetic `[C]`.** Receipt-owed: 14 + 2 + 1 + 3 + 1 + 13 + 3 + 2 + 6 = **45**. Refusals: **5**. No-model: **64**. Total 114. +**Capacity `[C]`.** E5 already holds 9 rows. These 10 make 19, the top of the G17 band, before any rerun. + +--- + +## §4 Execution (per row) + +| Phase | Do | Stop if | +|---|---|---| +| P0 | Ticket first: the cop mints it with `pmat work add`. Per-session worktree, never the primary checkout. Re-bind every `[U]` and `[A]` the row leans on with `pmat query`, at `origin/main`; state the tree in the receipt | a premise is false at HEAD: record `premise-falsified`, skip the row, report | +| P1 | Contract and falsifier first: one provable contract per row. The §9 plant for the row's invariants is RED before any engine line | a falsifier cannot be made RED: the gate is vacuous | +| P2 | Implement. Workspace crates only: trueno is `aprender-compute`, realizar is `aprender-serve`, entrenar is `aprender-train`. Never a crates.io `trueno` | — | +| P3 | Mutation RED then GREEN, observed **in the required check**, pasted in the PR body | green only locally | +| P4 | Quorum: one agy, one claude, one apr. Two non-passing rounds go to the human review queue | — | +| P5 | The agent arms its own merge through the merge queue. One PR from this spec in CI at a time. Receipt under `docs/audits/` | — | + +GPU development runs on lambda-labs or on yoga through existing make targets. sm_121 cells run on gx10. Hosts are provisioned only by `forjar apply -f machines//forjar.yaml` or a make target in paiml/infra. No ad-hoc SSH. + +--- + +## §5 Definitions of done (the exit bar) + +| # | Bar | Target | Probe | +|---|---|---|---| +| D1 | Registry commands with a verdict for the embedding kind | 114 of 114; unclassified 0 | the totality test beside `crates/apr-cli/tests/cli_commands.rs` | +| D2 | `receipt` cells green, per required host and claimed backend, at the cut sha | every cell the tag claims; unclaimed cells refuse with `removed_by` | `scripts/check_model_ladder.sh` | +| D3 | Refusals by name | 5 of 5, on their own line | probe receipt | +| D4 | Silent-wrong cells | **0** | I-3 plants | +| D5 | Files at parity | 6 of 6 Unsloth, 2 of 2 upstream as control | parity receipts | +| D6 | Parity threshold | a `models:` entry with basis, n and command | `evidence/parity/thresholds.yaml` | +| D7 | Default dimension 768 · non-finite values emitted · float16 dispatches · media inputs answered 200 · batch cells below threshold | 100% · 0 · 0 · 0 · 0 | I-4 to I-7 plants | +| D8 | Private attention loops added · quant `match` sites added | 0 · 0 | the two census commands in `docs/specifications/EPIC-0.74-any-model-plan.md` §6 | +| D9 | Fine-tune positive control · shuffled control · non-finite losses | 1 · 1 · 0 | EG-7 receipt | +| D10 | Dogfood | 1 corpus, 1 retrieval receipt | EG-9 receipt | +| D11 | Flow | ≤ 1 PR of this spec in CI; no PR older than 24 h | `gh pr list` | + +Throughput, quality retention and competitor comparisons have **no threshold**. They are recorded, then ratcheted after 3 records, each with its committed command. + +--- + +## §6 Budget and routing + +- Per row: K = ⌈1.1 · K̂⌉, andon at 0.8 · K. At andon: WIP commit, draft PR, `PARTIAL(andon)`, list what is unplanned. +- Must-carry set EG-0..EG-6: K̂ 730, K 803, andon 642. All ten rows: K̂ 1,180, K 1,298, andon 1,038. All `[A]`. +- Opus 5.5 takes cross-crate rows (EG-0, EG-2, EG-4, EG-7, EG-8). Sonnet 5.5 takes single-module rows. Fable is banned. +- Owner: the 0.72 look-ahead worker. It never competes with the current train for runners; the cop grants the CI slot. +- Backends beyond cpu and cuda follow whatever the ladder contract certifies at the cut. This spec adds none. + +--- + +## §7 STOP conditions (stop and report; never work around) + +| Stop | When | +|---|---| +| `pin-change` | a gate needs an oracle build newer than `build_commit` in `scripts/llama_pin.toml`. A pin move is a gate change and needs the operator's yes (Appendix C, D-3). EG-0 may measure against a provisional build; no row goes green in the release gate before the pin on `main` equals it | +| `epic-unbound` | no entry in `docs/roadmaps/epics.yaml` and no §1.5 row names this scope for 0.72 (D-1) | +| `projection-absent` | the 512 to 768 projection is not in the main GGUF and no declared sidecar carries it | +| `oracle-suspect` | the oracle disagrees with the card's stated behaviour, or degrades on its own F16 file. Diagnose the harness first; record, do not chase | +| `tokenizer-mismatch` | token ids differ from the oracle's on any corpus item | +| `op-unsupported-gpu` | EG-0 finds an op the capability contract marks GPU-unsupported. The CUDA cell refuses by name; it never falls back | +| `quorum-split` | after one amendment round | +| always | any step that needs a waiver, a threshold with no basis, new Python in the tree, a publish of a crate or of weights, a force-push, a re-tag, a ruleset or secret change, or a hand install on a host | + +Every stop names the other work that continues. + +--- + +## §8 Final report schema + +``` +APR-EMBED-001 | row= | verdict= | main= | PR= +verbs: classified /114 | receipt / | refuse /5 by name | silent-wrong <0> +files: /6 unsloth + /2 upstream | lambda= gx10= | backends +parity: oracle= min_cos= threshold= basis= | gpu-vs-cpu min_cos= | known-bad max_cos= +invar: dim768=/ nonfinite=<0> f16=<0> media200=<0> batch_below=<0> +train: method= band= gain= shuffled= | curve= +debt: private_attn=+<0> quant_match=+<0> lines= +unbound: [U] resolved / | filed +next: +``` + +--- + +## §9 Falsifiers (one per invariant; each plant is observed RED before the row may merge) + +| Invariant | Plant that must go RED | Control that must stay GREEN | +|---|---|---| +| I-1 | an EG diff that adds `cargo publish`, or `apr publish` without `--dry-run` | the publish preflight is byte-unchanged by every EG PR | +| I-2 | add a command to the registry with no verdict for one kind; add a kind with one verb undecided | the landed registry prints 114 classified. A scan of 0 commands is RED | +| I-3 | make `chat` on the embedding file exit 0 | `chat` on a Qwen rung stays green; `embed` on the file stays green | +| I-4 | return the pooled trunk state when 512 is requested; drop the re-normalisation after truncation | default is 768; the 256 response equals the leading 256 of the 768 response, re-normalised | +| I-5 | force float16 activations on the 2,000-token input | bf16 and f32 stay finite at 8,000 tokens | +| I-6 | mean-pool over padding positions | single against batched, and batch order permuted, stay above threshold; max \|Δ\| is recorded | +| I-7 | send three different images to the embeddings route: any 200 is RED | a text input returns its vector | +| I-8 | four known-bad pairs: a causal mask · the SiLU gate in place of GELU · the local-to-global pattern shifted by one layer · the document form in place of the query prompt. Each must fall below the threshold. One that does not means the threshold is too loose: raise it until it separates | `apr` against the oracle on the same file is above the threshold; the oracle against itself is recorded as the noise floor | +| I-9 | force 2 KV heads on a global layer; depth-prune one layer out of a period and export | the unpruned file loads; a whole-period prune round-trips | +| I-10 | a private attention loop in the new forward file; a quant-type `match` outside the type table | shared-block calls in the new file are above 0 | +| I-11 | train on shuffled pairs; leak held-out pairs into training | the clean run gains beyond the 3-seed band; the corpus guard reads the manifest | +| I-12 | a receipt with no model sha256; a receipt measured on an oracle build that is not the pin | a complete receipt is accepted | + +--- + +## Appendix A — verb × kind matrix for `kind = embedding` (proposal `[A]`; EG-0's quorum rules it) + +| Verdict | Count | Verbs | +|---|---|---| +| `receipt`, takes a model | 37 | `embed` `serve` · `inspect` `debug` `validate` `lint` `tensors` `trace` `diff` `hex` `tree` `flow` · `export` `convert` `stamp` `compile` `merge` `quantize` `rosetta` `shard` `encrypt` `decrypt` · `publish` · `finetune` `prune` `distill` `tune` · `bench` `eval` `check` `qa` `qualify` `compare-hf` `parity` · `profile` `ptx-map` · `diagnose` | +| `receipt`, model-adjacent | 8 | `pull` `import` `list` `rm` `unshard` `tokenize` `capability` `rag` | +| `refuse`, permanent | 5 | `run` `chat` `code` (no language-model head) · `rerank` (cross-encoder verb) · `showcase` (a Qwen demo). Each names `apr embed` | +| `no-model` | 64 | every other registry command | + +37 + 8 + 5 + 64 = 114. The 42 that take a model are the 37 and the 5. + +## Appendix B — adjacent tracks (not 0.72 rows) + +**Track J — Jev-Omni, a decision classifier on Gemma 4 12B `[X]`.** Open weights, Apache-2.0, safetensors BF16, tag `gemma4_unified`, a Python loader, CUDA required. It answers yes/no, choice and score questions with one probability per option. + +| Row | What | When | +|---|---|---| +| J-1 | APR-DECIDE-001 holds that the vendor model has no artifact sha, so no comparison is stateable `[A]` (Project copy). An open-weight model with the same interface has one. Add a CRUX decision contract row naming it as a competitor; its card numbers stay `[X]`; the comparator stays `NotRun{NoDeclaredExecutor}` until a forjar-declared leg exists | with APR-DECIDE-001 DEC-0; contract only | +| J-2 | `apr` runs it. Needs the Gemma 4 family (0 hits, G2), a third kind (`Classifier`), a head loader and the media towers. 12 B parameters × 2 bytes ≈ 24 GB `[C]`: too tight for a 24 GB card, fits gx10 | not before 0.74 (D-5) | + +**Track L — LLM2Vec, a recipe that turns a decoder into an encoder `[X]`.** Three steps: bidirectional attention, masked next-token prediction, contrastive training. + +| Row | What | When | +|---|---|---| +| L-1 | EG-7 builds the embedding objective (pooling, contrastive loss, bidirectional backward) with the backbone as a parameter. L-1 adds only the masked next-token objective (0 hits, G9) and the mask switch | one row under E8, 0.75 | +| L-2 | Qwen 3.5 is a hybrid of recurrent and attention layers `[A]` (APR-DECIDE-001, Project copy). A mask cannot make a recurrent layer bidirectional. First target: a dense attention rung the ladder already certifies. Qwen 3.5 gets a measured attention-layers-only arm, published whichever way it falls | after L-1 | + +## Appendix C — operator decisions (the default is what this spec assumes) + +| # | Decision | Default | +|---|---|---| +| D-1 | Where the 0.72 scope lives. `main` maps 0.72 to Agent Ready and excludes new model support (G13) | a new "Embeddings" entry on train 0.72 in `docs/roadmaps/epics.yaml`, budget 6 `[A]`, plus a 0.72 line in the §1.5 table (G14) naming EG-0..EG-6 | +| D-2 | The cut line | EG-0..EG-6 are must-carry: 34 receipt cells and 5 refusals. EG-7..EG-9 ship in the first train where their bar is green; until then their verbs refuse with `removed_by` | +| D-3 | The oracle pin (a gate change) | one pin, moved once at the 0.72 open to the upstream merge of PR 30054 or later. A bridge receipt measures the existing parity protocol at the old and the new pin back to back on one host, so ratios stay linkable. No second pin | +| D-4 | Images, audio, video | refuse by name in 0.72. `removed_by` is set when a GGUF that holds the encoders and an oracle that serves them both exist | +| D-5 | Tracks J and L; the architecture that 0.74's "real addition" measures | J-1 now, contract only. J-2 and L wait. Propose the Gemma 4 family as that addition: this model, Jev-Omni and the Gemma 4 chat files all need it | + +## Appendix D — five whys for the dominant risk (non-normative) + +1. Why would `apr` return a wrong vector with HTTP 200? The route pools the hidden state of whatever model is loaded (G5). +2. Why? The route does not know what kind of model it holds. +3. Why? No model-kind type exists. "Model" means causal language model in every verb. +4. Why was that never caught? All 8 ladder rungs are Qwen causal models (G12), and the capability registry derives a causal mask for every architecture (G6). +5. **Terminal mechanism:** two contracts state one kind's truth as universal — "dim == hidden size" and "always CausalMask" — so the wrong answer would also pass the contract. Countermeasure: I-2, I-4, and EG-0 items (d) and (e). + +## Appendix E — sources `[X]` + +- Unsloth GGUF: https://huggingface.co/unsloth/embeddinggemma-2-GGUF · upstream GGUF: https://huggingface.co/ggml-org/embeddinggemma-2-GGUF +- Unsloth model guide: https://unsloth.ai/docs/models/embeddinggemma-2 · embedding fine-tuning guide: https://unsloth.ai/docs/basics/embedding-finetuning +- Unsloth PR 12865: https://github.com/unslothai/unsloth/pull/12865 · Unsloth README: https://github.com/unslothai/unsloth/ +- llama.cpp PR 30054 (merged 2026-10-06; merge commit read as `4fbc76dec` — resolve it, do not copy it): https://github.com/ggml-org/llama.cpp/pull/30054 +- llama.cpp issue 30082: https://github.com/ggml-org/llama.cpp/issues/30082 +- Google developer guide (2026-10-06): https://developers.googleblog.com/embeddinggemma-2-the-developer-guide/ +- Jev-Omni card: https://huggingface.co/akhilaaa3/Jev-Omni · LLM2Vec: https://github.com/McGill-NLP/llm2vec From 89f464471bc1118d41955b82a593905956c2e1d3 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 7 Oct 2026 20:06:00 +0200 Subject: [PATCH 02/21] feat(embed): EG-0 f+g forward contract, pre-registered corpus, probe tooling - contracts/embeddinggemma2-forward-v1.yaml: local/global attention, gated GELU FFN, mean pool, projection, L2 normalise, truncation. Every obligation has an id; pv validate and pv lint report 0 errors, 0 warnings. - docs/audits/APR-EMBED-001/corpus: built by scripts/apr_embed_corpus.sh from the MIT book at a pinned commit; sha256 manifest; texts x 7 prompts + the document form, 5 lengths, batch-invariance set, retrieval micro-set, and train/held-out pairs split by chapter. - scripts/apr_embed_probe.sh + EG-0/probe-verbs.tsv: the 50-verb probe (a). Refs #4899 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/embeddinggemma2-forward-v1.yaml | 237 ++++++++++++++++++ .../audits/APR-EMBED-001/EG-0/probe-verbs.tsv | 53 ++++ docs/audits/APR-EMBED-001/corpus/SHA256SUMS | 10 + docs/audits/APR-EMBED-001/corpus/SOURCE | 3 + docs/audits/APR-EMBED-001/corpus/batch.tsv | 32 +++ .../APR-EMBED-001/corpus/finetune/heldout.tsv | 43 ++++ .../APR-EMBED-001/corpus/finetune/train.tsv | 179 +++++++++++++ docs/audits/APR-EMBED-001/corpus/lengths.tsv | 5 + docs/audits/APR-EMBED-001/corpus/prompts.tsv | 9 + .../APR-EMBED-001/corpus/retrieval/docs.tsv | 29 +++ .../APR-EMBED-001/corpus/retrieval/qrels.tsv | 16 ++ .../corpus/retrieval/queries.tsv | 16 ++ docs/audits/APR-EMBED-001/corpus/texts.tsv | 12 + scripts/apr_embed_corpus.sh | 135 ++++++++++ scripts/apr_embed_probe.sh | 47 ++++ 15 files changed, 826 insertions(+) create mode 100644 contracts/embeddinggemma2-forward-v1.yaml create mode 100644 docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/SHA256SUMS create mode 100644 docs/audits/APR-EMBED-001/corpus/SOURCE create mode 100644 docs/audits/APR-EMBED-001/corpus/batch.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/finetune/heldout.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/finetune/train.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/lengths.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/prompts.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/retrieval/docs.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/retrieval/qrels.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/retrieval/queries.tsv create mode 100644 docs/audits/APR-EMBED-001/corpus/texts.tsv create mode 100755 scripts/apr_embed_corpus.sh create mode 100755 scripts/apr_embed_probe.sh diff --git a/contracts/embeddinggemma2-forward-v1.yaml b/contracts/embeddinggemma2-forward-v1.yaml new file mode 100644 index 0000000000..50f22b1cdc --- /dev/null +++ b/contracts/embeddinggemma2-forward-v1.yaml @@ -0,0 +1,237 @@ +metadata: + version: 1.0.0 + created: '2026-10-07' + author: PAIML Engineering + description: EmbeddingGemma 2 forward pass -- tokens to a unit-norm, projected, optionally truncated embedding (APR-EMBED-001 EG-0 f) + references: + - docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md (I-4, I-5, I-6, I-9, I-10) + - 'Kusupati et al. (2022) Matryoshka Representation Learning' + - 'Beltagy et al. (2020) Longformer: The Long-Document Transformer (sliding-window attention)' + depends_on: + - bidirectional-attention-v1 + - sliding-window-attention-v1 + - gelu-kernel-v1 + - embedding-lookup-v1 +equations: + layer_attention_kind: + formula: kind(l) = global if shape(attn_q[l]) equals the widest q projection in the file, else local + domain: l in [0, block_count), tensor shapes and metadata read from the GGUF + codomain: '{local, global}' + invariants: + - The kind of every layer is read from tensor shapes or metadata, never from layer-index arithmetic or an architecture constant (I-9) + - At least one layer is global and at least one is local + preconditions: + - block_count > 0 + - every block has an attn_q tensor + lean_theorem: Theorems.EmbeddingGemma2_Layer_Kind + postconditions: + - 'every layer has exactly one kind and the kind set is {local, global}' + local_attention: + formula: A_local(x)_i = softmax_j( q_i . k_j / sqrt(d_head) + M_ij ) v_j with M_ij = 0 if |i - j| < W else -inf + domain: x in R^{n x d_model}, W = sliding_window read from metadata + codomain: R^{n x d_model} + invariants: + - The mask is symmetric (bidirectional); no causal term (attention.causal = false) + - Positions with |i - j| >= W receive exactly zero weight + preconditions: + - W > 0 + - n > 0 + lean_theorem: Theorems.EmbeddingGemma2_Local_Attention + postconditions: + - 'output.shape == input.shape and all output values are finite' + global_attention: + formula: A_global(x)_i = softmax_j( q_i . k_j / sqrt(d_head) + P_ij ) v_j with P_ij = 0 for real tokens, -inf for padding + domain: x in R^{n x d_model} + codomain: R^{n x d_model} + invariants: + - Every real token attends to every real token in both directions + - Padding positions receive zero weight + preconditions: + - n > 0 + lean_theorem: Theorems.EmbeddingGemma2_Global_Attention + postconditions: + - 'output.shape == input.shape and all output values are finite' + gated_gelu_ffn: + formula: FFN(h) = W_down( GELU_tanh(W_gate h) * (W_up h) ) + domain: h in R^{n x d_model} + codomain: R^{n x d_model} + invariants: + - The activation is GELU (tanh approximation), never SiLU; a gated MLP does not imply SwiGLU + preconditions: + - ffn_gate, ffn_up and ffn_down are present in every block + lean_theorem: Theorems.EmbeddingGemma2_Gated_Gelu_Ffn + postconditions: + - 'output.shape == input.shape and all output values are finite' + mean_pool: + formula: p = (1 / |R|) * sum_{i in R} H_i, R = indices of real (non-padding) tokens + domain: H in R^{n x d_model}, |R| >= 1 + codomain: R^{d_model} + invariants: + - Padding rows never contribute + - The result does not depend on batch size or on the text's batch position (I-6) + preconditions: + - pooling_type read from metadata is mean + - '|R| >= 1' + lean_theorem: Theorems.EmbeddingGemma2_Mean_Pool + postconditions: + - 'output.len() == d_model' + projection: + formula: e = W_out p, W_out = output.weight read from the file + domain: p in R^{d_model} + codomain: R^{d_out}, d_out = embedding_length_out read from metadata + invariants: + - The returned vector is the projected vector, never the pooled trunk state (I-4) + - A file without the projection tensor is refused by name, never served at trunk width + preconditions: + - output.weight exists with shape [d_model, d_out] + lean_theorem: Theorems.EmbeddingGemma2_Projection + postconditions: + - 'output.len() == embedding_length_out' + l2_normalise: + formula: u = e / ||e||_2 + domain: e in R^{d}, ||e||_2 > 0 + codomain: R^{d}, ||u||_2 = 1 + invariants: + - '| ||u||_2 - 1 | <= 1e-5' + - Every component is finite (I-5) + preconditions: + - '||e||_2 > 0 and finite' + lean_theorem: Theorems.EmbeddingGemma2_L2_Normalise + postconditions: + - 'output.len() == embedding_length_out' + truncation: + formula: t_k = u[0..k] / ||u[0..k]||_2, k in {768, 512, 256, 128} + domain: u in R^{d_out}, k <= d_out + codomain: R^{k}, ||t_k||_2 = 1 + invariants: + - t_k equals the leading k components of the full projected vector, re-normalised (I-4) + - t_k for k = d_model (512) is not the pooled trunk state + preconditions: + - k is one of the declared Matryoshka dimensions + lean_theorem: Theorems.EmbeddingGemma2_Truncation + postconditions: + - 'output.len() == k and abs(norm(output) - 1) <= 1e-5' +proof_obligations: +- id: EG2-INV-001 + type: invariant + property: Layer kind from the file + formal: kind(l) is a function of tensor shapes and metadata only; renumbering layers in a synthetic file moves the kinds with the tensors + applies_to: all +- id: EG2-INV-002 + type: invariant + property: Local attention window + formal: weight(i, j) == 0 for |i - j| >= W in every local layer + applies_to: all +- id: EG2-INV-003 + type: invariant + property: Bidirectional global attention + formal: for real tokens i, j in a global layer, weight(i, j) > 0 and weight(j, i) > 0 + applies_to: all +- id: EG2-INV-004 + type: invariant + property: Gated GELU activation + formal: FFN uses GELU_tanh on the gate branch; substituting SiLU changes the output beyond tolerance + applies_to: all +- id: EG2-INV-005 + type: invariant + property: Mean pool over real tokens only + formal: appending padding to a sequence leaves p unchanged within 1e-6 + applies_to: all +- id: EG2-INV-006 + type: invariant + property: Projected output + formal: len(e) == embedding_length_out and e == W_out p + applies_to: all +- id: EG2-BND-007 + type: bound + property: Unit norm and finite + formal: '| ||u||_2 - 1 | <= 1e-5 and is_finite(u[i]) for all i' + applies_to: all +- id: EG2-INV-008 + type: invariant + property: Truncation re-normalises the leading dims + formal: t_k == normalise(u_full[0..k]) for each k in {768, 512, 256, 128} + applies_to: all +- id: EG2-INV-009 + type: invariant + property: Batch and order invariance + formal: embed(x) == embed_batch([.., x, ..])[pos] within 1e-5 for every batch size and position + applies_to: all +- id: EG2-EQ-010 + type: equivalence + property: Oracle parity + formal: cosine(apr(x), oracle(x)) >= threshold for every corpus item, threshold from the thresholds file's known-good and known-bad method + applies_to: all +falsification_tests: +- id: FALSIFY-EG2-001 + rule: Layer kind from the file + prediction: A synthetic file whose global and local blocks are swapped yields swapped layer kinds + test: unit test over a synthetic tensor-shape table, no layer-index arithmetic in the classifier + if_fails: Layer kind derived from an index pattern or an architecture constant (I-9) +- id: FALSIFY-EG2-002 + rule: Local attention window + prediction: Attention weights beyond the window are exactly zero in local layers + test: proptest over n in [1, 3W] with random q, k; inspect the weight matrix + if_fails: Window off by one, or a causal mask applied instead of a symmetric one +- id: FALSIFY-EG2-003 + rule: Bidirectional global attention + prediction: Reversing token order permutes, but does not change the set of, attention weights in global layers + test: proptest with random input, compare weight matrices under reversal + if_fails: A causal mask leaked into an embedding-kind model +- id: FALSIFY-EG2-004 + rule: Gated GELU activation + prediction: The FFN output matches the GELU_tanh reference and differs from a SiLU substitute + test: unit test against fixed vectors for both activations + if_fails: Gated MLP mapped to SwiGLU +- id: FALSIFY-EG2-005 + rule: Mean pool over real tokens only + prediction: Padding a text to batch length leaves its pooled vector unchanged + test: proptest with random hidden states and random padding lengths + if_fails: Padding rows counted in the mean +- id: FALSIFY-EG2-006 + rule: Projected output + prediction: Default response is embedding_length_out (768) long and not equal to the pooled trunk state + test: corpus item through the forward path; compare length and compare with the unprojected pool + if_fails: Projection skipped (the 512-long trunk vector is unit norm and passes a length check at 512) +- id: FALSIFY-EG2-007 + rule: Unit norm and finite + prediction: Every output vector has norm 1 within 1e-5 and no NaN or Inf + test: every corpus item at every declared dimension + if_fails: Missing normalisation, or a float16 activation overflow (I-5) +- id: FALSIFY-EG2-008 + rule: Truncation re-normalises the leading dims + prediction: The 256 response equals the leading 256 of the 768 response, re-normalised + test: compare responses at 768, 512, 256 and 128 for every corpus item + if_fails: Truncation without re-normalisation, or the 512 response taken from the trunk +- id: FALSIFY-EG2-009 + rule: Batch and order invariance + prediction: The batch-invariance corpus set embeds identically alone and at every batch position + test: corpus batch-invariance set at batch 1, 8 and 32, every position + if_fails: Cross-row leakage through padding, pooling or attention masks +- id: FALSIFY-EG2-010 + rule: Oracle parity + prediction: Cosine with the oracle's golden vectors meets the threshold on every corpus item + test: compare with goldens written by the oracle build on the same file + if_fails: Weight loading, tokenizer or architecture mismatch +kani_harnesses: +- id: KANI-EG2-001 + obligation: Unit norm and finite + property: L2 normalisation of a finite non-zero vector has norm 1 within tolerance + bound: 4 + strategy: stub_float + solver: cadical + harness: verify_l2_unit_norm +- id: KANI-EG2-002 + obligation: Truncation re-normalises the leading dims + property: Truncated output equals the re-normalised prefix of the full vector + bound: 4 + strategy: stub_float + solver: cadical + harness: verify_truncation_prefix +qa_gate: + id: F-EG2F-001 + name: embeddinggemma2-forward-v1 Contract + description: Quality gate for the EmbeddingGemma 2 forward pass + checks: + - validation + - falsification diff --git a/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv b/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv new file mode 100644 index 0000000000..fe0cc74724 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv @@ -0,0 +1,53 @@ +# APR-EMBED-001 EG-0(a): one probe per verb. Columns: verbargs. +# {M}=model under test {B}=BF16 sibling {O}=fresh scratch dir. Every probe runs with +# stdin /SOURCE, else HEAD +# outdir default: docs/audits/APR-EMBED-001/corpus +# Check: (cd docs/audits/APR-EMBED-001/corpus && sha256sum -c SHA256SUMS) +set -euo pipefail + +OUT="${2:-docs/audits/APR-EMBED-001/corpus}" +if [ -n "${1:-}" ]; then + COMMIT="$1" +elif [ -f "${OUT}/SOURCE" ]; then + COMMIT="$(sed -n 's/^commit=//p' "${OUT}/SOURCE")" +else + COMMIT="$(git rev-parse HEAD)" +fi +COMMIT="$(git rev-parse "${COMMIT}^{commit}")" +mkdir -p "${OUT}" + +# Every book chapter at the pinned commit, in byte order (LC_ALL=C). +FILES="$(git ls-tree -r --name-only "${COMMIT}" -- book/src | grep '\.md$' | grep -v 'SUMMARY\.md$' | LC_ALL=C sort)" +if [ -z "${FILES}" ]; then + echo "apr_embed_corpus: no book chapters at ${COMMIT}" >&2 + exit 1 +fi + +# prose : one chapter as "Hheading" and "Pprose line" records. +# Code fences, tables, HTML, images and link targets are dropped. +prose() { + git show "${COMMIT}:$1" | awk ' + /^```/ { fence = !fence; next } + fence { next } + /^\|/ || /^#]/, "", l); gsub(/\t/, " ", l) + sub(/^[[:space:]]*([-+]|[0-9]+\.)[[:space:]]+/, "", l) + gsub(/[[:space:]]+/, " ", l); sub(/^ /, "", l); sub(/ $/, "", l) + if (l != "") print "P\t" l + }' +} + +# Sections: one row per "## heading" whose body has >= 40 words and whose heading +# has >= 3 words (generic one- and two-word headings do not identify a passage). +# split = cksum(path) mod 10: 0 retrieval, 1-2 heldout, 3-9 train. Split by FILE, +# so no chapter feeds two splits. +SECTIONS="$(mktemp)" +trap 'rm -f "${SECTIONS}"' EXIT +while IFS= read -r f; do + bucket=$(( $(printf '%s' "${f}" | cksum | cut -d' ' -f1) % 10 )) + case "${bucket}" in 0) split=retrieval ;; 1|2) split=heldout ;; *) split=train ;; esac + prose "${f}" | awk -F'\t' -v f="${f}" -v s="${split}" ' + function flush() { + if (h != "" && split(h, hw, " ") >= 3 && split(b, bw, " ") >= 40) { + n = 0; body = "" + for (i = 1; i <= length(bw) && n < 120; i++) { body = body (n ? " " : "") bw[i]; n++ } + print s "\t" f "\t" h "\t" body + } + h = ""; b = "" + } + $1 == "H" { flush(); h = $2; next } + $1 == "P" && h != "" { b = b " " $2 } + END { flush() }' +done <<< "${FILES}" > "${SECTIONS}" + +# retrieval/: queries (heading), docs (document form inputs), qrels (1:1). +# At most 16 queries; every retrieval-split body is a doc, so distractors exist. +mkdir -p "${OUT}/retrieval" "${OUT}/finetune" +awk -F'\t' '$1 == "retrieval"' "${SECTIONS}" | awk -F'\t' -v o="${OUT}/retrieval" ' + { n++; id = sprintf("r%03d", n) + print id "\t" $3 "\t" $4 > (o "/docs.tsv") + if (n <= 16) { print "q" substr(id, 2) "\t" $3 > (o "/queries.tsv"); print "q" substr(id, 2) "\t" id > (o "/qrels.tsv") } }' + +# finetune/: train and held-out (query, positive passage) pairs, disjoint by file. +awk -F'\t' '$1 == "train" { n++; printf "t%04d\t%s\t%s\n", n, $3, $4 }' "${SECTIONS}" > "${OUT}/finetune/train.tsv" +awk -F'\t' '$1 == "heldout" { n++; printf "h%04d\t%s\t%s\n", n, $3, $4 }' "${SECTIONS}" > "${OUT}/finetune/heldout.tsv" + +# texts.tsv: 12 (title, text) items for the 7 prompts x document-form matrix, +# taken from the retrieval split at 60 words each. +awk -F'\t' '$1 == "retrieval" && n < 12 { n++; split($4, w, " "); t = "" + for (i = 1; i <= 60 && i <= length(w); i++) t = t (i > 1 ? " " : "") w[i] + printf "x%02d\t%s\t%s\n", n, $3, t }' "${SECTIONS}" > "${OUT}/texts.tsv" + +# batch.tsv: 32 items of mixed length (3, 17, 60, 120 words), so padding differs +# across a batch of 8 or 32. +awk -F'\t' '$1 == "train" && n < 32 { split("3 17 60 120", L, " "); k = L[(n % 4) + 1]; n++ + split($4, w, " "); t = "" + for (i = 1; i <= k && i <= length(w); i++) t = t (i > 1 ? " " : "") w[i] + printf "b%02d\t%s\n", n, t }' "${SECTIONS}" > "${OUT}/batch.tsv" + +# lengths.tsv: running book prose cut at word counts that target 8, 128, 1024, +# 2000 and 8000 tokens. The exact token count of each item is the oracle's, +# recorded with the goldens (EG-0 c); the target here is a label, not a claim. +while IFS= read -r f; do prose "${f}"; done <<< "${FILES}" | awk -F'\t' '$1 == "P" { print $2 }' \ + | tr '\n' ' ' | awk -v RS=' ' 'NF { w[++n] = $0 } END { + split("8 128 1024 2000 8000", T, " "); split("6 96 780 1530 6100", W, " ") + for (j = 1; j <= 5; j++) { t = "" + for (i = 1; i <= W[j]; i++) t = t (i > 1 ? " " : "") w[i] + printf "len%05d\t%d\t%s\n", T[j], W[j], t } }' > "${OUT}/lengths.tsv" + +# prompts.tsv: the model card's seven task prompts and the document form (M5). +{ + printf 'name\tquery_template\tdocument_template\n' + printf 'SearchQuery\ttask: search result | query: {content}\ttitle: {title} | text: {content}\n' + printf 'QuestionAnswering\ttask: question answering | query: {content}\ttitle: {title} | text: {content}\n' + printf 'FactChecking\ttask: fact checking | query: {content}\ttitle: {title} | text: {content}\n' + printf 'CodeRetrieval\ttask: code retrieval | query: {content}\ttitle: {title} | text: {content}\n' + printf 'Classification\ttask: classification | query: {content}\t-\n' + printf 'Clustering\ttask: clustering | query: {content}\t-\n' + printf 'SentenceSimilarity\ttask: sentence similarity | query: {content}\t-\n' + printf 'Document\t-\ttitle: none | text: {content}\n' +} > "${OUT}/prompts.tsv" + +printf 'commit=%s\nsource=book/src (MIT, this repository)\ngenerator=scripts/apr_embed_corpus.sh\n' "${COMMIT}" > "${OUT}/SOURCE" + +# A corpus with an empty part measured nothing: fail rather than write it. +for p in texts.tsv batch.tsv lengths.tsv retrieval/docs.tsv retrieval/queries.tsv finetune/train.tsv finetune/heldout.tsv; do + if [ ! -s "${OUT}/${p}" ]; then + echo "apr_embed_corpus: ${p} is empty" >&2 + exit 1 + fi +done + +(cd "${OUT}" && find . -type f ! -name SHA256SUMS | LC_ALL=C sort | sed 's|^\./||' | xargs sha256sum > SHA256SUMS) +echo "apr_embed_corpus: ${COMMIT} -> ${OUT}" diff --git a/scripts/apr_embed_probe.sh b/scripts/apr_embed_probe.sh new file mode 100755 index 0000000000..f6fa573e94 --- /dev/null +++ b/scripts/apr_embed_probe.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# apr_embed_probe.sh - APR-EMBED-001 EG-0(a): run every verb in a probe table +# against one model file with the tree-pinned `apr`, recording the exit code and +# the first non-empty output line of each. Measurement only: no verdict, no gate. +# +# Usage: bash scripts/apr_embed_probe.sh +# Table: docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv +set -euo pipefail + +MODEL="${1:?model file}" +BF16="${2:?bf16 sibling}" +OUT="${3:?output tsv}" +TABLE="docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv" +TIMEOUT_S="${APR_EMBED_PROBE_TIMEOUT:-120}" + +. scripts/apr_bin.sh || exit 1 + +SCRATCH="$(mktemp -d)" +trap 'rm -rf "${SCRATCH}"' EXIT + +printf '# apr=%s model_sha256=%s timeout_s=%s\n' \ + "$("${APR}" --version)" "$(sha256sum "${MODEL}" | cut -d' ' -f1)" "${TIMEOUT_S}" > "${OUT}" +printf 'verb\texit\tfirst_line\n' >> "${OUT}" + +n=0 +while IFS=$'\t' read -r verb args; do + case "${verb}" in ''|'#'*) continue ;; esac + o="${SCRATCH}/${verb}" + mkdir -p "${o}" + args="${args//\{M\}/${MODEL}}" + args="${args//\{B\}/${BF16}}" + args="${args//\{O\}/${o}}" + # Word-splitting of the substituted args is intended: the table holds argv. + # shellcheck disable=SC2086 + rc=0 + timeout "${TIMEOUT_S}" "${APR}" "${verb}" ${args} < /dev/null > "${o}.log" 2>&1 || rc=$? + first="$(grep -m1 -vE '^[[:space:]]*$|^[━─╭╰│]+$' "${o}.log" | tr '\t' ' ' | cut -c1-200 || true)" + printf '%s\t%s\t%s\n' "${verb}" "${rc}" "${first}" >> "${OUT}" + n=$((n + 1)) +done < "${TABLE}" + +# A probe that ran zero verbs measured nothing. +if [ "${n}" -eq 0 ]; then + echo "apr_embed_probe: 0 verbs probed from ${TABLE}" >&2 + exit 1 +fi +echo "apr_embed_probe: ${n} verbs -> ${OUT}" From 1a2727c2a7baf2d4c0e5d5bacba0c3d5c4093151 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 7 Oct 2026 20:32:04 +0200 Subject: [PATCH 03/21] =?UTF-8?q?feat(embed):=20EG-0=20a,b,d,e=20=E2=80=94?= =?UTF-8?q?=20probe=20receipt,=20M9=20resolved,=20verb=20totality=20guard,?= =?UTF-8?q?=20embedding=20ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (a) Probe of 50 verbs on the UD-Q4_K_XL file, exit code and first line each. (b) inspect + tensors on all 8 files; m9.md gives every M9 value with its command (F16-on-oracle pending (c)). (d) apr-cli-commands-v1.yaml: every command declares one verdict per model kind (receipt / refuse / no-model), refusals name the verb to use. The totality guard is two tests in the existing cli_commands target (fragment 050, so ci / gate via workspace-test); mutation RED receipt in d-totality-mutation.md. (e) apr-model-capability-v1.yaml (+ mirror): eight embedding ops, all gpu_supported false with a reason; CausalMask moves from `always` to `by_kind.generative`. RequiredOp gains the eight variants; nothing derives them yet (EG-1/EG-2). D-1: epic E10 Embeddings on 0.72 (budget 6) and the 0.72 §1.5 row. Refs #4899 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/apr-cli-commands-v1.yaml | 120 + contracts/apr-model-capability-v1.yaml | 75 +- .../contracts/apr-model-capability-v1.yaml | 75 +- crates/apr-cli/tests/cli_commands.rs | 266 ++ crates/aprender-serve/src/capability.rs | 36 +- .../APR-EMBED-001/EG-0/a-probe-udq4.tsv | 52 + ...gml-org-embeddinggemma-2-BF16.inspect.json | 57 + ...gml-org-embeddinggemma-2-BF16.tensors.json | 3531 +++++++++++++++++ ...gml-org-embeddinggemma-2-Q8_0.inspect.json | 57 + ...gml-org-embeddinggemma-2-Q8_0.tensors.json | 3531 +++++++++++++++++ ...unsloth-embeddinggemma-2-BF16.inspect.json | 59 + ...unsloth-embeddinggemma-2-BF16.tensors.json | 3531 +++++++++++++++++ .../unsloth-embeddinggemma-2-F16.inspect.json | 59 + .../unsloth-embeddinggemma-2-F16.tensors.json | 3531 +++++++++++++++++ ...unsloth-embeddinggemma-2-Q8_0.inspect.json | 59 + ...unsloth-embeddinggemma-2-Q8_0.tensors.json | 3531 +++++++++++++++++ ...h-embeddinggemma-2-UD-Q4_K_XL.inspect.json | 63 + ...h-embeddinggemma-2-UD-Q4_K_XL.tensors.json | 3531 +++++++++++++++++ ...h-embeddinggemma-2-UD-Q5_K_XL.inspect.json | 63 + ...h-embeddinggemma-2-UD-Q5_K_XL.tensors.json | 3531 +++++++++++++++++ ...h-embeddinggemma-2-UD-Q6_K_XL.inspect.json | 63 + ...h-embeddinggemma-2-UD-Q6_K_XL.tensors.json | 3531 +++++++++++++++++ .../APR-EMBED-001/EG-0/d-totality-mutation.md | 23 + docs/audits/APR-EMBED-001/EG-0/m9.md | 88 + .../audits/APR-EMBED-001/EG-0/probe-verbs.tsv | 14 +- docs/roadmaps/epics.yaml | 1 + .../APR-RELEASE-001-train-and-build-kaizen.md | 1 + scripts/apr_embed_probe.sh | 7 +- scripts/capability_op_impl_map.txt | 8 + 29 files changed, 29478 insertions(+), 16 deletions(-) create mode 100644 docs/audits/APR-EMBED-001/EG-0/a-probe-udq4.tsv create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.inspect.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.tensors.json create mode 100644 docs/audits/APR-EMBED-001/EG-0/d-totality-mutation.md create mode 100644 docs/audits/APR-EMBED-001/EG-0/m9.md diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index 65489d4bdf..79b74a2bba 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -56,12 +56,15 @@ commands: category: inference description: "Run model directly (auto-download, cache, execute)" requires_model: true + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} side_effects: [network, filesystem] - name: serve category: inference description: "Inference server (plan/run)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [network, filesystem] subcommands: [plan, run] @@ -69,12 +72,16 @@ commands: category: inference description: "Interactive chat with language model" requires_model: true + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} side_effects: [network, filesystem] - name: code category: inference description: "Sovereign AI coding assistant" requires_model: true + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} side_effects: [network, filesystem] # ── Model Inspection (read-only, no side effects) ── @@ -82,12 +89,14 @@ commands: category: inspection description: "Inspect model metadata, vocab, and structure" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: debug category: inspection description: "Simple debugging output" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] subcommands: [embed-viz] @@ -95,42 +104,49 @@ commands: category: inspection description: "Validate model integrity and quality" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: validate-manifest category: inspection description: "Verify publish manifest (F-PUBLISH-MANIFEST-001): FALSIFY-PM-001..006" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: lint category: inspection description: "Check for best practices and conventions" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: beat-run category: inspection description: "Evaluate a BeatBenchmark contract against a measured value (PMAT-741)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: manifest category: model_ops description: "Emit SHA-256 manifest of input files (CRUX-G-05)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] - name: explain category: inspection description: "Explain errors, architecture, tensors, and kernel dispatch" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: tensors category: inspection description: "List tensor names and shapes" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] # aprender#2377 finding 3: the producers `*-lint` help documents. @@ -139,6 +155,7 @@ commands: category: inspection description: "Dataset inspection tools (audio-inspect)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] subcommands: [audio-inspect] @@ -146,6 +163,7 @@ commands: category: analysis description: "Kernel-level parity measurements (parity)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] subcommands: [parity] @@ -153,30 +171,35 @@ commands: category: inspection description: "Layer-by-layer trace analysis" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: diff category: inspection description: "Compare two models; --quant-roundtrip surfaces per-tensor quant error (CRUX-B-20)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: hex category: inspection description: "Format-aware binary forensics" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: tree category: inspection description: "Model architecture tree view" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: flow category: inspection description: "Data flow visualization" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] # ── Model Operations (transforms, creates files) ── @@ -184,48 +207,56 @@ commands: category: model_ops description: "Export model to other formats" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: import category: model_ops description: "Import from external formats" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [network, filesystem] - name: convert category: model_ops description: "Convert/optimize model" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: stamp category: model_ops description: "Stamp provenance fields (license, data_source, data_license) onto an existing .apr file" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: compile category: model_ops description: "Compile model into standalone executable" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: merge category: model_ops description: "Merge multiple models" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: quantize category: model_ops description: "Quantize model weights" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: rosetta category: model_ops description: "Rosetta Stone - Universal model format converter" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] # ── Model Registry ── @@ -235,24 +266,28 @@ commands: category: registry description: "Download and cache model from HuggingFace" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [network, filesystem] - name: list category: registry description: "List cached models" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: rm category: registry description: "Remove model from cache" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: registry category: registry description: "Registry operations (CRUX-A-01): inspect alias map, etc." requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] subcommands: [aliases] @@ -260,6 +295,7 @@ commands: category: registry description: "Publish model to HuggingFace Hub" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [network] # ── Training ── @@ -267,24 +303,28 @@ commands: category: training description: "Fine-tune model with LoRA/QLoRA" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem, gpu] - name: prune category: training description: "Prune model (structured/unstructured)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: distill category: training description: "Knowledge distillation (teacher -> student)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem, gpu] - name: train category: training description: "Training pipeline (plan/apply)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem, gpu] subcommands: [apply, archive, cluster-status, halving, plan, submit, sweep, watch] @@ -292,12 +332,14 @@ commands: category: training description: "Pretrain small language models from scratch (MODEL-2 loop)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem, gpu] - name: tokenize category: training description: "Tokenizer training pipeline (plan/apply)" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] subcommands: [apply, encode, encode-corpus, import-hf, plan, repair-manifest, train] @@ -305,6 +347,7 @@ commands: category: training description: "ML tuning: LoRA/QLoRA configuration, memory planning" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] # ── Evaluation & QA ── @@ -312,36 +355,42 @@ commands: category: evaluation description: "Benchmark throughput" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: eval category: evaluation description: "Evaluate model perplexity or classification metrics" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: check category: evaluation description: "Model self-test: 10-stage pipeline integrity check" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: qa category: evaluation description: "Falsifiable QA checklist for model releases" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: qualify category: evaluation description: "Cross-subcommand smoke test" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: canary category: evaluation description: "Manage canary tests for regression" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [check, create] @@ -349,12 +398,14 @@ commands: category: evaluation description: "Compare APR model against HuggingFace source" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [network] - name: parity category: evaluation description: "GPU/CPU parity check" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [gpu] # ── GPU & Profiling ── @@ -362,35 +413,41 @@ commands: category: hardware description: "GPU status and VRAM reservation management" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [gpu] - name: capability category: inspection description: What this build can and cannot do, and why (from apr-model-capability-v1) requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: devices category: hardware description: "Backend discovery: probe, enumerate, print (PP-066 R-0)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: profile category: hardware description: "Deep profiling with Roofline analysis" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: ptx category: hardware description: "PTX analysis and bug detection" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: ptx-debug category: hardware description: "PTX falsification analysis and FKR test generation (was the aprender-ptx-debug binary)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [analyze, gen-fkr, version] @@ -398,12 +455,14 @@ commands: category: hardware description: "Model-to-PTX source mapping" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: cbtop category: hardware description: "ComputeBrick pipeline monitor" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] # ── Data & Pipeline ── @@ -411,6 +470,7 @@ commands: category: data description: "Data quality pipeline" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [audit, balance, decontaminate, dedup, split, x] @@ -418,6 +478,7 @@ commands: category: data description: "Pipeline orchestration (plan/apply/status)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] # ── TUI & Monitoring ── @@ -427,18 +488,21 @@ commands: category: ui description: "Interactive terminal UI" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: monitor category: ui description: "Attach live TUI to a running training session" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: runs category: ui description: "List, show, and compare training experiment runs" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] subcommands: [diff, ls, show] @@ -446,6 +510,7 @@ commands: category: ui description: "Interactive experiment browser" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] # ── Miscellaneous ── @@ -455,6 +520,8 @@ commands: category: misc description: "Qwen2.5-Coder showcase demo" requires_model: true + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} side_effects: [network, filesystem] - name: modelfile @@ -462,6 +529,7 @@ commands: description: | CRUX-K-11: parse Ollama-style Modelfile DSL into apr config JSON. requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] # parse — parse a Modelfile and emit config as JSON or human-readable text subcommands: [parse] @@ -489,6 +557,7 @@ commands: (GH-876 Milestone 2). The rest land as they are delegated to the probador library. requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem, network] # tensor — export tensor activations for visual regression testing (PMAT-481) # llm — benchmark an OpenAI-compatible endpoint (GH-876 Milestone 2); @@ -500,258 +569,303 @@ commands: category: misc description: "Automated Five Whys diagnosis on a training checkpoint" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [] - name: ollama-chat-lint category: inspection description: "Lint a captured Ollama /api/chat response (CRUX-C-04)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: ollama-tools-lint category: inspection description: "Lint a captured Ollama /api/chat function-calling response (CRUX-I-04)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: dry-sampling-lint category: inspection description: "Lint a captured DRY-sampling observation (CRUX-C-23)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: awq-lint category: inspection description: "Lint a captured AWQ quality/compression/flags observation (CRUX-B-08)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: fp8-lint category: inspection description: "Lint a captured FP8 round-trip + SM-capability observation (CRUX-B-11)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: nf4-lint category: inspection description: "Lint a captured NF4 codebook/roundtrip/storage/parity observation (CRUX-B-10)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: oom-lint category: inspection description: "Lint a captured CUDA OOM postmortem report (CRUX-F-13)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: tool-use-lint category: inspection description: "Lint a captured OpenAI tool-use response (CRUX-C-11)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: gbnf-lint category: inspection description: "Lint a GBNF grammar-constrained observation (CRUX-C-10)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: typical-p-lint category: inspection description: "Lint a captured typical-p sampling observation (CRUX-C-22)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: registry-quota-lint category: inspection description: "Lint a captured registry byte-quota observation (CRUX-A-22)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: imatrix-lint category: inspection description: "Lint a captured imatrix calibration observation (CRUX-B-07)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: gptq-lint category: inspection description: "Lint a captured GPTQ compression/cosine/flags observation (CRUX-B-09)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: embeddings-lint category: inspection description: "Lint a captured /v1/embeddings observation (CRUX-C-13)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: unified-search-lint category: inspection description: "Lint a captured Hub+local unified-search merge observation (CRUX-A-23)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: rm-gc-lint category: inspection description: "Lint a captured `apr rm` / externally captured gc blob-GC observation (CRUX-A-25)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: shared-cache-lint category: inspection description: "Lint a captured APR_MODELS shared-cache observation (CRUX-A-21)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: ppl category: inspection description: "Perplexity classifier (CRUX-E-02): PPL = exp(-mean(log p))" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: quant-preservation-lint category: inspection description: "Validate dequant→requant metadata preservation (CRUX-B-19): general.* + tokenizer.* must be byte-equal modulo {quantization_version, file_type}" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: prometheus-lint category: inspection description: "Validate captured Prometheus /metrics body (CRUX-K-07): 0.0.4 text-format parse + optional Content-Type + optional K-07 required-metric set" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: otlp-lint category: inspection description: "Validate captured OTLP/JSON ExportTraceServiceRequest (CRUX-K-08): apr.inference span present + gen_ai.*/apr.tokens.* attributes + W3C trace-context propagation" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: kv-timeline-lint category: inspection description: "Validate captured KV-cache utilization timeline (CRUX-F-06): schema + block conservation + used_pct arithmetic + peak consistency + preemption-threshold trigger" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: gpu-memtrace-lint category: inspection description: "Validate captured GPU memory Chrome Trace JSON (CRUX-F-07): schema + alloc/free pairing by addr + per-(pid,tid) monotonic timestamps" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: explain-token-lint category: inspection description: "Validate an externally captured token-selection JSONL sampler-chain trace (CRUX-F-19; no apr producer yet): schema + Σ post_prob ≈ 1 + sampled_id present in candidates + optional greedy=argmax assertion" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: check-finite-lint category: inspection description: "Validate externally captured check-finite outputs (CRUX-F-11; no apr producer yet): error JSON shape (6 keys + value ∈ {nan,+inf,-inf}) + layer-coverage list (min_layers + transformer-op prefixes)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: attn-viz-lint category: inspection description: "Validate an externally captured attention dump (CRUX-F-17; no apr producer yet): 4-D attention dump row-softmax normalization + causal-mask zero + HTML heatmap count" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: embed-viz-lint category: inspection description: "Validate an externally captured embedding-projection CSV (CRUX-F-18; no apr producer yet): schema (4 columns + finite coords + non-negative token_id) + optional row-count=vocab_size + optional byte-identical determinism check" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: nccl-diag-lint category: inspection description: "Validate captured NCCL failure-diagnostics JSON (CRUX-F-15): 6 required keys (host, rank, nccl_version, last_op, code, suggest) + optional exit-code >= 128 + optional nvidia.com doc-link in suggest" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: react-trace-lint category: inspection description: "Validate an externally captured ReAct loop trace (CRUX-I-06; no apr producer yet): termination reason matches exit-code (final_answer=0, max_iterations|timeout=2, tool_error=3, parse_fail=4) + iteration bound + scratchpad grammar (Thought/Action/Observation/Final Answer)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: hang-trace-lint category: inspection description: "Validate captured `$APR_TRACE_DIR` after a hang/timeout (CRUX-F-14): per-rank rank{N}.py.txt files (world_size + non-empty) OR empty-on-success + exit-code 124/1/0 distinction" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: ddp-metrics-lint category: inspection description: "Validate two externally captured DDP metrics JSONs, N=1 and N=k (CRUX-D-11; no apr producer yet): scaling efficiency T_N/(N*T_1) >= 0.85 + loss parity |L_N - L_1|/L_1 <= 0.01 + per-step allreduce bandwidth > 0" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: audio-inspect-lint category: inspection description: "Validate an externally captured audio-inspect JSON body (CRUX-H-13; no apr producer yet): amplitude bounds (finite + [-1,1]) + sample_rate (canonical set or expected) + channel_shape (positive + optional channel-count match)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: attn-parity-lint category: inspection description: "Validate captured flash-attn2 parity + provenance JSON outputs (CRUX-L-02): max_abs_diff <= 5e-3, cosine_sim >= 0.9999, kernel_source matches hf-kernels-community:flash-attn2@<40-hex>, fallback reason present when not flash2, head_dim error names the rejection" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: rerank category: inference description: "Score a (query, passage) pair with a BERT cross-encoder loaded from an APR file (GH-326 Phase 3 + 3b). Two input modes: (1) pre-tokenised `--input-ids` + `--token-type-ids` u32 lists; (2) Phase 3b text mode — `--query` + `--passage` + `--vocab ` invokes in-process WordPiece tokenisation that builds `[CLS] query [SEP] passage [SEP]` with `token_type_ids = 0` for the query side and `1` for the passage side. Emits sigmoid-mapped relevance ∈ [0,1] by default or `--raw-logit` for the raw classifier logit. Supports `--json` output." requires_model: true + kinds: {generative: refuse, embedding: refuse} + use_instead: {generative: run, embedding: embed} side_effects: [] - name: embed category: inference description: "Produce sentence embeddings from a BERT bi-encoder loaded from an APR file (GH-326 Phase 6). Loads encoder-only `BertModel` checkpoints (e.g. `sentence-transformers/all-MiniLM-L6-v2`), tokenises text with WordPiece, runs the encoder forward, then pools hidden states with `--pool cls` ([CLS] hidden state) or `--pool mean` (mean of token positions; sentence-transformers default). Optionally L2-normalises (`--normalize`, default true). Repeatable `--text` for batch encoding. Outputs `{ text, embedding: [f32; hidden_dim], dim }` per input." requires_model: true + kinds: {generative: refuse, embedding: receipt} + use_instead: {generative: run} side_effects: [] - name: shard category: model_ops description: "Split safetensors file into shards + weight-map index (CRUX-B-05)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: unshard category: model_ops description: "Reconstruct single safetensors file from sharded directory (CRUX-B-05)" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: oracle category: misc description: "RAG oracle — search Sovereign AI Stack documentation" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: grad-norm category: inspection description: "Gradient-norm telemetry classifier (CRUX-F-09)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [] - name: encrypt category: model_ops description: "Encrypt .apr model file (AES-256-GCM)" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: decrypt category: model_ops description: "Decrypt .apr model file" requires_model: true + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] - name: mcp category: misc description: "Run MCP (Model Context Protocol) server over stdio" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [stdio] # ── Consolidated sibling CLIs (APR-MONO) ── @@ -762,6 +876,7 @@ commands: category: pipeline description: "RAG pipeline: index, query, transcribe (was the trueno-rag binary)" requires_model: false + kinds: {generative: receipt, embedding: receipt} side_effects: [filesystem] subcommands: [demo, extract-frames, index, info, query, transcribe] @@ -769,6 +884,7 @@ commands: category: tools description: "zram device management (was the trueno-zram binary)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [benchmark, create, remove, status] @@ -776,6 +892,7 @@ commands: category: tools description: "Discrete-event simulation (was the simular binary)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] # `version` comes from #2527's simular clap conversion, which the batch # takes over #2493's. FALSIFY-CLI-006 caught the omission. @@ -785,6 +902,7 @@ commands: category: tools description: "Compute-graph profiling (was the aprender-cgp binary)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [baseline, bench, compete, contract, diff, doctor, explain, profile, roofline, trace, tui] @@ -792,6 +910,7 @@ commands: category: tools description: "Provable-contracts CLI (also ships as the standalone pv binary)" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [filesystem] subcommands: [audit, book, census, certify, challenge, check-parity, codegen, coq, coverage, diff, discharge, equations, explain, extract, extract-pytorch, flux, fuzz, generate, graph, infer, invariants, kaizen, kani, lean, lean-status, lint, migrate, mirai, obligations, ontology, pipeline, probar, proof-status, query, roofline, scaffold, score, status, tla, unlock, validate, verify-bindings, verify-pipeline, verify-structure] @@ -801,6 +920,7 @@ commands: category: tools description: "Print apr's own command surface as JSON (apr-cli-surface/v1, #3745); hidden gate input" requires_model: false + kinds: {generative: no-model, embedding: no-model} side_effects: [stdio] # ── Falsification Conditions ── diff --git a/contracts/apr-model-capability-v1.yaml b/contracts/apr-model-capability-v1.yaml index 31753fa830..b50b4fa33d 100644 --- a/contracts/apr-model-capability-v1.yaml +++ b/contracts/apr-model-capability-v1.yaml @@ -127,6 +127,7 @@ spec: > # from `gpu_supported_ops()`: the eight `ops.insert(...)` lines are `true`, and # the five named in the trailing comment are `false` WITH the comment's own # reason and citation, which is the point — a negative fact stops being prose. +# The eight APR-EMBED-001 rows below them are `false` for the same reason. ops: - op: RoPE gpu_supported: true @@ -166,6 +167,56 @@ ops: CUDA forward applies only the two LLaMA-style norms, not the four-per-block Gemma2/Gemma3 norms, so those fall back to CPU citation: PMAT-824 + # APR-EMBED-001 EG-0 (e): the embedding kind's ops, read from the + # EmbeddingGemma 2 files (tensor names and metadata in + # docs/audits/APR-EMBED-001/EG-0/b/). No backend implements any of them yet, + # so none is GPU-supported and no derivation in Rust produces them (EG-1/EG-2). + - op: BidirectionalMask + gpu_supported: false + reason: > + attention.causal = false. The CUDA attention kernels fold in a causal + mask; no symmetric-mask path exists + citation: APR-EMBED-001 + - op: SlidingWindowAttn + gpu_supported: false + reason: > + local layers attend within attention.sliding_window (1024) with their own + RoPE base (rope.freq_base_swa); no windowed attention path exists + citation: APR-EMBED-001 + - op: GatedGeluMlp + gpu_supported: false + reason: > + gate GELU_tanh(W_gate h) * (W_up h) -> W_down. The gated GPU path is + SwiGLU (SiLU); mapping a gated MLP to SwiGLU is the wrong answer + citation: APR-EMBED-001 + - op: PerLayerInput + gpu_supported: false + reason: > + per-layer input gating (blk.N.inp_gate, blk.N.proj, per_layer_model_proj, + per_layer_proj_norm, blk.N.post_norm); no backend implements it + citation: APR-EMBED-001 + - op: LayerOutputScale + gpu_supported: false + reason: > + blk.N.layer_output_scale multiplies each block's output; no backend + implements it + citation: APR-EMBED-001 + - op: MeanPool + gpu_supported: false + reason: > + pooling_type = 1 (mean over real tokens); no pooling kernel exists + citation: APR-EMBED-001 + - op: DenseProjection + gpu_supported: false + reason: > + output.weight projects the pooled embedding_length (512) to + embedding_length_out (768); no pooled-projection path exists + citation: APR-EMBED-001 + - op: L2Normalize + gpu_supported: false + reason: > + the response is unit-norm; no normalise kernel exists on the response path + citation: APR-EMBED-001 # ── op → implementing symbol, and whether anything calls it ────────────────── # Three states, because they are three different facts and only the first can be @@ -449,8 +500,16 @@ quant_types: reason: not named by the GPU whitelist # ── how an architecture's required ops are derived ─────────────────────────── -# Transcribed from `required_ops()` / `required_ops_for_model()`. Declared so a -# consumer can compute the same set without re-reading Rust. +# The generative rows are transcribed from `required_ops()` / +# `required_ops_for_model()`. Declared so a consumer can compute the same set +# without re-reading Rust. +# +# APR-EMBED-001: the mask is a property of the model KIND, not of every +# transformer. "always CausalMask" stated one kind's truth as universal, so an +# embedding model would have passed the contract with the wrong mask. The +# embedding rows are declared AHEAD of Rust: no loader admits the embedding kind +# yet (EG-1), so `required_ops()` still inserts CausalMask unconditionally, which +# is correct for the only kind it can see. op_derivation: positional_encoding: Rope: [RoPE] @@ -468,7 +527,17 @@ op_derivation: conditional: has_bias: [BiasAdd] has_qk_norm: [QkNorm] - always: [CausalMask] + always: [] + by_kind: + generative: [CausalMask] + embedding: [BidirectionalMask, MeanPool, DenseProjection, L2Normalize] + # Read from the file (APR-EMBED-001 I-9), never from an architecture name. + per_file: + sliding_window: [SlidingWindowAttn] # attention.sliding_window > 0 + gated_gelu_ffn: [GatedGeluMlp] # ffn_gate present and the activation is GELU + inp_gate: [PerLayerInput] # blk.N.inp_gate present + layer_output_scale: [LayerOutputScale] # blk.N.layer_output_scale present + post_attention_norm: [PostAttnFfnNorm] # blk.N.post_attention_norm present per_architecture: softcap_postnorm: [AttnFinalSoftcap, PostAttnFfnNorm] diff --git a/crates/apr-cli/contracts/apr-model-capability-v1.yaml b/crates/apr-cli/contracts/apr-model-capability-v1.yaml index 31753fa830..b50b4fa33d 100644 --- a/crates/apr-cli/contracts/apr-model-capability-v1.yaml +++ b/crates/apr-cli/contracts/apr-model-capability-v1.yaml @@ -127,6 +127,7 @@ spec: > # from `gpu_supported_ops()`: the eight `ops.insert(...)` lines are `true`, and # the five named in the trailing comment are `false` WITH the comment's own # reason and citation, which is the point — a negative fact stops being prose. +# The eight APR-EMBED-001 rows below them are `false` for the same reason. ops: - op: RoPE gpu_supported: true @@ -166,6 +167,56 @@ ops: CUDA forward applies only the two LLaMA-style norms, not the four-per-block Gemma2/Gemma3 norms, so those fall back to CPU citation: PMAT-824 + # APR-EMBED-001 EG-0 (e): the embedding kind's ops, read from the + # EmbeddingGemma 2 files (tensor names and metadata in + # docs/audits/APR-EMBED-001/EG-0/b/). No backend implements any of them yet, + # so none is GPU-supported and no derivation in Rust produces them (EG-1/EG-2). + - op: BidirectionalMask + gpu_supported: false + reason: > + attention.causal = false. The CUDA attention kernels fold in a causal + mask; no symmetric-mask path exists + citation: APR-EMBED-001 + - op: SlidingWindowAttn + gpu_supported: false + reason: > + local layers attend within attention.sliding_window (1024) with their own + RoPE base (rope.freq_base_swa); no windowed attention path exists + citation: APR-EMBED-001 + - op: GatedGeluMlp + gpu_supported: false + reason: > + gate GELU_tanh(W_gate h) * (W_up h) -> W_down. The gated GPU path is + SwiGLU (SiLU); mapping a gated MLP to SwiGLU is the wrong answer + citation: APR-EMBED-001 + - op: PerLayerInput + gpu_supported: false + reason: > + per-layer input gating (blk.N.inp_gate, blk.N.proj, per_layer_model_proj, + per_layer_proj_norm, blk.N.post_norm); no backend implements it + citation: APR-EMBED-001 + - op: LayerOutputScale + gpu_supported: false + reason: > + blk.N.layer_output_scale multiplies each block's output; no backend + implements it + citation: APR-EMBED-001 + - op: MeanPool + gpu_supported: false + reason: > + pooling_type = 1 (mean over real tokens); no pooling kernel exists + citation: APR-EMBED-001 + - op: DenseProjection + gpu_supported: false + reason: > + output.weight projects the pooled embedding_length (512) to + embedding_length_out (768); no pooled-projection path exists + citation: APR-EMBED-001 + - op: L2Normalize + gpu_supported: false + reason: > + the response is unit-norm; no normalise kernel exists on the response path + citation: APR-EMBED-001 # ── op → implementing symbol, and whether anything calls it ────────────────── # Three states, because they are three different facts and only the first can be @@ -449,8 +500,16 @@ quant_types: reason: not named by the GPU whitelist # ── how an architecture's required ops are derived ─────────────────────────── -# Transcribed from `required_ops()` / `required_ops_for_model()`. Declared so a -# consumer can compute the same set without re-reading Rust. +# The generative rows are transcribed from `required_ops()` / +# `required_ops_for_model()`. Declared so a consumer can compute the same set +# without re-reading Rust. +# +# APR-EMBED-001: the mask is a property of the model KIND, not of every +# transformer. "always CausalMask" stated one kind's truth as universal, so an +# embedding model would have passed the contract with the wrong mask. The +# embedding rows are declared AHEAD of Rust: no loader admits the embedding kind +# yet (EG-1), so `required_ops()` still inserts CausalMask unconditionally, which +# is correct for the only kind it can see. op_derivation: positional_encoding: Rope: [RoPE] @@ -468,7 +527,17 @@ op_derivation: conditional: has_bias: [BiasAdd] has_qk_norm: [QkNorm] - always: [CausalMask] + always: [] + by_kind: + generative: [CausalMask] + embedding: [BidirectionalMask, MeanPool, DenseProjection, L2Normalize] + # Read from the file (APR-EMBED-001 I-9), never from an architecture name. + per_file: + sliding_window: [SlidingWindowAttn] # attention.sliding_window > 0 + gated_gelu_ffn: [GatedGeluMlp] # ffn_gate present and the activation is GELU + inp_gate: [PerLayerInput] # blk.N.inp_gate present + layer_output_scale: [LayerOutputScale] # blk.N.layer_output_scale present + post_attention_norm: [PostAttnFfnNorm] # blk.N.post_attention_norm present per_architecture: softcap_postnorm: [AttnFinalSoftcap, PostAttnFfnNorm] diff --git a/crates/apr-cli/tests/cli_commands.rs b/crates/apr-cli/tests/cli_commands.rs index 939ab9ade5..5fa4cdaf75 100644 --- a/crates/apr-cli/tests/cli_commands.rs +++ b/crates/apr-cli/tests/cli_commands.rs @@ -1114,3 +1114,269 @@ fn falsify_2607_piped_prompt_is_not_refused_as_a_bare_invocation() { model resolution and stop at NO_MODEL — proof the guard let it through. stderr:\n{stderr}" ); } + +// ── APR-EMBED-001 I-2: kind totality ───────────────────────────────────────── +// +// Every registry command declares exactly one verdict per model kind, and the +// verdict is one of three states. There is no fourth state: a typo, an +// "advisory" or a missing kind is a violation, never a pass. + +const MODEL_KINDS: [&str; 2] = ["generative", "embedding"]; +const VERDICTS: [&str; 3] = ["receipt", "refuse", "no-model"]; + +fn name_of(cmd: &serde_yaml::Value) -> &str { + cmd.get("name") + .and_then(|n| n.as_str()) + .unwrap_or("") +} + +fn verdict_of(cmd: &serde_yaml::Value, kind: &str) -> Option { + cmd.get("kinds")?.get(kind)?.as_str().map(str::to_string) +} + +/// The registry's `commands:` list, or the violation that stops the scan. +fn registry_commands(yaml: &str) -> Result, String> { + let doc: serde_yaml::Value = + serde_yaml::from_str(yaml).map_err(|e| format!("registry does not parse: {e}"))?; + let commands = doc + .get("commands") + .and_then(|c| c.as_sequence()) + .ok_or_else(|| "registry has no `commands:` list".to_string())?; + if commands.is_empty() { + return Err("0 commands scanned".to_string()); + } + Ok(commands.clone()) +} + +/// A refusal must name a registered verb that takes the kind; any other verdict +/// must name none. +fn use_instead_violation( + cmd: &serde_yaml::Value, + kind: &str, + verdict: &str, + commands: &[serde_yaml::Value], +) -> Option { + let name = name_of(cmd); + let instead = cmd + .get("use_instead") + .and_then(|u| u.get(kind)) + .and_then(|u| u.as_str()); + match (verdict, instead) { + ("refuse", None) => Some(format!( + "{name}: refuses kind `{kind}` without naming the verb to use (use_instead)" + )), + ("refuse", Some(other)) => match commands.iter().find(|c| name_of(c) == other) { + None => Some(format!( + "{name}: use_instead.{kind} names `{other}`, which is not a registered command" + )), + Some(t) if verdict_of(t, kind).as_deref() != Some("receipt") => Some(format!( + "{name}: use_instead.{kind} names `{other}`, which does not take kind `{kind}`" + )), + Some(_) => None, + }, + (_, Some(other)) => Some(format!( + "{name}: use_instead.{kind} = `{other}` on a `{verdict}` verdict" + )), + (_, None) => None, + } +} + +/// Violations for one kind of one command. Returns the verdict when it is valid. +fn kind_violations( + cmd: &serde_yaml::Value, + kind: &str, + commands: &[serde_yaml::Value], + bad: &mut Vec, +) -> Option { + let name = name_of(cmd); + let Some(v) = verdict_of(cmd, kind) else { + bad.push(format!("{name}: no verdict for kind `{kind}`")); + return None; + }; + if !VERDICTS.contains(&v.as_str()) { + bad.push(format!( + "{name}: kind `{kind}` has verdict `{v}`, not one of {VERDICTS:?}" + )); + return None; + } + let requires_model = cmd.get("requires_model").and_then(|r| r.as_bool()) == Some(true); + if v == "no-model" && requires_model { + bad.push(format!( + "{name}: requires_model is true but kind `{kind}` says no-model" + )); + } + bad.extend(use_instead_violation(cmd, kind, &v, commands)); + Some(v) +} + +/// Violations for one command across every kind. +fn command_violations( + cmd: &serde_yaml::Value, + commands: &[serde_yaml::Value], + bad: &mut Vec, +) { + let name = name_of(cmd); + let Some(kinds) = cmd.get("kinds").and_then(|k| k.as_mapping()) else { + bad.push(format!("{name}: no `kinds:` map")); + return; + }; + for key in kinds.keys() { + let key = key.as_str().unwrap_or(""); + if !MODEL_KINDS.contains(&key) { + bad.push(format!("{name}: unknown model kind `{key}`")); + } + } + let verdicts: Vec = MODEL_KINDS + .iter() + .filter_map(|kind| kind_violations(cmd, kind, commands, bad)) + .collect(); + let no_model = verdicts.iter().filter(|v| *v == "no-model").count(); + if verdicts.len() == MODEL_KINDS.len() && no_model > 0 && no_model < verdicts.len() { + bad.push(format!("{name}: no-model for one kind but not the other")); + } +} + +/// All kind-totality violations in a registry document. Returns the violations +/// and the names of the commands scanned; a scan of zero commands is a violation. +fn kind_totality_violations(yaml: &str) -> (Vec, Vec) { + let commands = match registry_commands(yaml) { + Ok(c) => c, + Err(e) => return (vec![e], Vec::new()), + }; + let mut bad = Vec::new(); + for cmd in &commands { + command_violations(cmd, &commands, &mut bad); + } + let names: std::collections::BTreeSet = + commands.iter().map(|c| name_of(c).to_string()).collect(); + (bad, names.into_iter().collect()) +} + +#[test] +fn every_command_declares_one_verdict_per_model_kind() { + let Some(text) = provable_contracts::workspace_file_or_skip!( + "every_command_declares_one_verdict_per_model_kind", + "contracts/apr-cli-commands-v1.yaml" + ) else { + return; + }; + let (bad, scanned) = kind_totality_violations(&text); + assert!( + bad.is_empty(), + "APR-EMBED-001 I-2 kind totality:\n {}", + bad.join("\n ") + ); + let missed: Vec<&str> = registered_commands() + .into_iter() + .filter(|c| !scanned.iter().any(|s| s == c)) + .collect(); + assert!( + missed.is_empty(), + "the totality scan must cover every registered command; not scanned: {missed:?}" + ); +} + +#[test] +fn kind_totality_guard_case_table() { + const OK: &str = "commands: + - name: run + requires_model: true + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} + - name: embed + requires_model: true + kinds: {generative: refuse, embedding: receipt} + use_instead: {generative: run} + - name: pull + requires_model: false + kinds: {generative: receipt, embedding: receipt} + - name: explain + requires_model: false + kinds: {generative: no-model, embedding: no-model} +"; + let (bad, scanned) = kind_totality_violations(OK); + assert!(bad.is_empty(), "must-pass case flagged: {bad:?}"); + assert_eq!(scanned.len(), 4); + + // (case, mutation applied to OK, substring the violation must contain) + let must_fail: &[(&str, &str, &str, &str)] = &[ + ("zero commands", "", "commands: []\n", "0 commands"), + ("no commands list", "", "binary: apr\n", "no `commands:`"), + ( + "kinds missing", + " kinds: {generative: no-model, embedding: no-model}\n", + "", + "no `kinds:`", + ), + ( + "kind missing", + "{generative: no-model, embedding: no-model}", + "{generative: no-model}", + "no verdict for kind `embedding`", + ), + ( + "fourth state", + "{generative: no-model, embedding: no-model}", + "{generative: no-model, embedding: advisory}", + "advisory", + ), + ( + "third kind", + "{generative: no-model, embedding: no-model}", + "{generative: no-model, embedding: no-model, classifier: refuse}", + "unknown model kind", + ), + ( + "model verb says no-model", + "{generative: receipt, embedding: refuse}", + "{generative: no-model, embedding: refuse}", + "requires_model is true", + ), + ( + "refuse names nothing", + " use_instead: {embedding: embed}\n", + "", + "without naming", + ), + ( + "refuse names unknown verb", + "use_instead: {embedding: embed}", + "use_instead: {embedding: embedx}", + "not a registered command", + ), + ( + "refuse names a refusing verb", + "use_instead: {generative: run}", + "use_instead: {generative: embed}", + "does not take kind", + ), + ( + "use_instead on a receipt", + "kinds: {generative: receipt, embedding: receipt}\n", + "kinds: {generative: receipt, embedding: receipt}\n use_instead: {embedding: run}\n", + "on a `receipt` verdict", + ), + ( + "mixed no-model", + "{generative: receipt, embedding: receipt}", + "{generative: receipt, embedding: no-model}", + "no-model for one kind", + ), + ]; + for (case, from, to, expect) in must_fail { + let doc = if from.is_empty() { + (*to).to_string() + } else { + assert!( + OK.contains(from), + "case `{case}`: mutation anchor not in OK" + ); + OK.replacen(from, to, 1) + }; + let (bad, _) = kind_totality_violations(&doc); + assert!( + bad.iter().any(|b| b.contains(expect)), + "case `{case}` must be RED with `{expect}`; got {bad:?}" + ); + } +} diff --git a/crates/aprender-serve/src/capability.rs b/crates/aprender-serve/src/capability.rs index a6ec2573d8..ef500383b5 100644 --- a/crates/aprender-serve/src/capability.rs +++ b/crates/aprender-serve/src/capability.rs @@ -63,6 +63,24 @@ pub enum RequiredOp { /// support this op; a model requiring it is routed to CPU at the capability /// layer. PostAttnFfnNorm, + // APR-EMBED-001 EG-0 (e): the embedding kind's ops. Declared so the contract + // and this enum stay one set; nothing derives or implements them yet (EG-1/2). + /// Bidirectional attention mask (embedding kind; `attention.causal = false`) + BidirectionalMask, + /// Sliding-window local attention with its own RoPE base (window read from the file) + SlidingWindowAttn, + /// Gated GELU FFN: GELU_tanh(gate) ⊙ up → down. Not SwiGLU + GatedGeluMlp, + /// Per-layer input gating (`inp_gate`, `proj`, `per_layer_model_proj`) + PerLayerInput, + /// Per-block output scale (`layer_output_scale`) + LayerOutputScale, + /// Mean pool over real tokens + MeanPool, + /// Dense projection of the pooled state to the declared output width + DenseProjection, + /// L2 normalisation of the response vector + L2Normalize, } /// Every `RequiredOp` variant, in declaration order. @@ -73,7 +91,7 @@ pub enum RequiredOp { /// variant added here without a contract row fails a test rather than becoming a /// silent hole — the op-shaped form of #3850's "an unknown quant is decoded as /// Q4_K". -pub const ALL_REQUIRED_OPS: [RequiredOp; 13] = [ +pub const ALL_REQUIRED_OPS: [RequiredOp; 21] = [ RequiredOp::RoPE, RequiredOp::GQA, RequiredOp::MHA, @@ -87,6 +105,14 @@ pub const ALL_REQUIRED_OPS: [RequiredOp; 13] = [ RequiredOp::CausalMask, RequiredOp::AttnFinalSoftcap, RequiredOp::PostAttnFfnNorm, + RequiredOp::BidirectionalMask, + RequiredOp::SlidingWindowAttn, + RequiredOp::GatedGeluMlp, + RequiredOp::PerLayerInput, + RequiredOp::LayerOutputScale, + RequiredOp::MeanPool, + RequiredOp::DenseProjection, + RequiredOp::L2Normalize, ]; impl std::fmt::Display for RequiredOp { @@ -105,6 +131,14 @@ impl std::fmt::Display for RequiredOp { Self::CausalMask => write!(f, "CausalMask"), Self::AttnFinalSoftcap => write!(f, "AttnFinalSoftcap"), Self::PostAttnFfnNorm => write!(f, "PostAttnFfnNorm"), + Self::BidirectionalMask => write!(f, "BidirectionalMask"), + Self::SlidingWindowAttn => write!(f, "SlidingWindowAttn"), + Self::GatedGeluMlp => write!(f, "GatedGeluMlp"), + Self::PerLayerInput => write!(f, "PerLayerInput"), + Self::LayerOutputScale => write!(f, "LayerOutputScale"), + Self::MeanPool => write!(f, "MeanPool"), + Self::DenseProjection => write!(f, "DenseProjection"), + Self::L2Normalize => write!(f, "L2Normalize"), } } } diff --git a/docs/audits/APR-EMBED-001/EG-0/a-probe-udq4.tsv b/docs/audits/APR-EMBED-001/EG-0/a-probe-udq4.tsv new file mode 100644 index 0000000000..ce4636e9a1 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/a-probe-udq4.tsv @@ -0,0 +1,52 @@ +# apr=apr 0.70.1 (f745e7abc) model_sha256=ea905fd08e8061db77a0031cbb5096d459d7cfd2bdd0ad2ba66e483f16719493 timeout_s=120 +verb exit first_line +run 8 === APR Run === +serve 0 ━━━ Inference Plan: embeddinggemma-2-UD-Q4_K_XL (Q4_K) on CUDA GPU (unknown) ━━━ +chat 0 === Model Chat (GGUF Format) === +code 1 Launched apr serve on port 42191 (pid 3047862, --gpu) +inspect 0 Rosetta Stone Inspection +debug 0 embeddinggemma-2-UD-Q4_K_XL.gguf: GGUF gemma-embedding2 (413) +validate 0 Validating /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf... +lint 0 Model Lint +tensors 0 Tensors: /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf +trace 0 Layer Trace: /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf +diff 0 Model Diff +hex 0 GGUF Binary Forensics: /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf +tree 0 embeddinggemma-2-UD-Q4_K_XL.gguf (413 tensors, 152.5 MB) +flow 0 Model: embeddinggemma-2-UD-Q4_K_XL.gguf (decoder-only (LLaMA/Qwen2), GGUF) +export 5 APR Export +convert 0 APR Convert +stamp 5 Reading /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf +compile 4 error: Invalid APR format: Only APR v2 format (APR\0) is supported for compilation. Convert with `apr import` first. +merge 0 APR Merge — Plan +quantize 0 APR Quantize — Plan +rosetta 0 Rosetta Stone Inspection +publish 0 === DRY RUN: Would publish to example/embeddinggemma-2 === +finetune 0 === apr finetune (GH-244: LoRA/QLoRA Fine-tuning) === +prune 0 APR Prune — Plan +distill 0 APR Distill — Plan +tune 0 === apr tune (GH-176: ML Tuning via entrenar-lora) === +bench 5 === APR Benchmark === +eval 5 === APR Evaluation === +check 5 === Model Self-Test (PMAT-112: Real Validation) === +qa 5 APR Quality Assurance +qualify 5 Qualify +compare-hf 5 Loading local model: /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf (Gguf) +parity 9 error: Feature not enabled: cuda feature required for parity check +profile 5 === apr profile (Real Per-Operation Telemetry) === +ptx-map 0 Scope: PTX kernel DISPATCH map — verifies kernel launch configuration +showcase 5 ╔═══════════════════════════════════════════════════════════════╗ +diagnose 0 SSC Training Diagnosis (Five Whys) +rerank 5 error: Validation failed: Failed to read vocab /rerank/vocab.txt: No such file or directory (os error 2) +embed 5 error: Validation failed: Failed to parse APR v2 at /unsloth/embeddinggemma-2-UD-Q4_K_XL.gguf: InvalidMagic([71, 71, 85, 70]) +shard 5 error: Validation failed: apr shard failed: safetensors error: HeaderTooLarge +encrypt 5 Enter passphrase (or set ALBOR_ENCRYPT_KEY env var): +decrypt 5 Enter passphrase (or set ALBOR_ENCRYPT_KEY env var): +pull 0 === APR Pull === +import 5 APR Import Pipeline +list 0 === Cached Models === +rm 3 === APR Remove === +unshard 5 error: Validation failed: apr unshard failed: input is not a directory: /unshard/shards +tokenize 0 29104 +capability 0 GPU-supported operations +rag 0 Trueno-RAG Pipeline diff --git a/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.inspect.json new file mode 100644 index 0000000000..c57b64a56e --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.inspect.json @@ -0,0 +1,57 @@ +{ + "file": "embeddinggemma-2-BF16.gguf", + "format": "GGUF", + "file_size": 557950176, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "BF16", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.file_type": "32", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "embeddinggemma-2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.tensors.json new file mode 100644 index 0000000000..5d48f851d8 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-BF16.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-BF16.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 542158944, + "tensors": [ + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "BF16", + "size_bytes": 268435456 + }, + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "BF16", + "size_bytes": 786432 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "BF16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.inspect.json new file mode 100644 index 0000000000..3a1d302029 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.inspect.json @@ -0,0 +1,57 @@ +{ + "file": "embeddinggemma-2-Q8_0.gguf", + "format": "GGUF", + "file_size": 309855456, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "Q8_0", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.file_type": "7", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "embeddinggemma-2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.tensors.json new file mode 100644 index 0000000000..7f1e572ab3 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/ggml-org-embeddinggemma-2-Q8_0.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-Q8_0.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 294064224, + "tensors": [ + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "Q8_0", + "size_bytes": 417792 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "BF16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "Q8_0", + "size_bytes": 142606336 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.inspect.json new file mode 100644 index 0000000000..e8b13f0fc2 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.inspect.json @@ -0,0 +1,59 @@ +{ + "file": "embeddinggemma-2-BF16.gguf", + "format": "GGUF", + "file_size": 557950240, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "BF16", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "32", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.tensors.json new file mode 100644 index 0000000000..5d48f851d8 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-BF16.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-BF16.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 542158944, + "tensors": [ + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "BF16", + "size_bytes": 268435456 + }, + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "BF16", + "size_bytes": 786432 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "BF16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "BF16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "BF16", + "size_bytes": 524288 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "BF16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.inspect.json new file mode 100644 index 0000000000..6c33f46ad2 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.inspect.json @@ -0,0 +1,59 @@ +{ + "file": "embeddinggemma-2-F16.gguf", + "format": "GGUF", + "file_size": 557950240, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "F16", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "1", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.tensors.json new file mode 100644 index 0000000000..efd4b26987 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-F16.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-F16.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 542158944, + "tensors": [ + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "F16", + "size_bytes": 268435456 + }, + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "F16", + "size_bytes": 786432 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "F16", + "size_bytes": 2097152 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "F16", + "size_bytes": 1048576 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "F16", + "size_bytes": 524288 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "F16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.inspect.json new file mode 100644 index 0000000000..8d42dacfb6 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.inspect.json @@ -0,0 +1,59 @@ +{ + "file": "embeddinggemma-2-Q8_0.gguf", + "format": "GGUF", + "file_size": 309855520, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "Q8_0", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "7", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.tensors.json new file mode 100644 index 0000000000..7f1e572ab3 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-Q8_0.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-Q8_0.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 294064224, + "tensors": [ + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "Q8_0", + "size_bytes": 417792 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "BF16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "Q8_0", + "size_bytes": 142606336 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q8_0", + "size_bytes": 557056 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q8_0", + "size_bytes": 1114112 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.inspect.json new file mode 100644 index 0000000000..75aeef24d1 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.inspect.json @@ -0,0 +1,63 @@ +{ + "file": "embeddinggemma-2-UD-Q4_K_XL.gguf", + "format": "GGUF", + "file_size": 175673856, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "Q4_K", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "15", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "quantize.imatrix.chunks_count": "230", + "quantize.imatrix.dataset": "data/calib.txt", + "quantize.imatrix.entries_count": "216", + "quantize.imatrix.file": "outputs/eg2/gguf/work/imatrix.gguf", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.tensors.json new file mode 100644 index 0000000000..57db00cacc --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q4_K_XL.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-UD-Q4_K_XL.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 159882336, + "tensors": [ + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "Q6_K", + "size_bytes": 322560 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "Q5_K", + "size_bytes": 4325376 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "Q4_K", + "size_bytes": 75497472 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q4_K", + "size_bytes": 294912 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 147456 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q4_K", + "size_bytes": 589824 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.inspect.json new file mode 100644 index 0000000000..fbec334f02 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.inspect.json @@ -0,0 +1,63 @@ +{ + "file": "embeddinggemma-2-UD-Q5_K_XL.gguf", + "format": "GGUF", + "file_size": 210055680, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "Q5_K", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "17", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "quantize.imatrix.chunks_count": "230", + "quantize.imatrix.dataset": "data/calib.txt", + "quantize.imatrix.entries_count": "216", + "quantize.imatrix.file": "outputs/eg2/gguf/work/imatrix.gguf", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.tensors.json new file mode 100644 index 0000000000..10997aef0f --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q5_K_XL.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-UD-Q5_K_XL.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 194264160, + "tensors": [ + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "Q6_K", + "size_bytes": 322560 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "Q6_K", + "size_bytes": 5160960 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "Q5_K", + "size_bytes": 92274688 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q5_K", + "size_bytes": 360448 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 180224 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q5_K", + "size_bytes": 720896 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.inspect.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.inspect.json new file mode 100644 index 0000000000..ce1e7842c6 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.inspect.json @@ -0,0 +1,63 @@ +{ + "file": "embeddinggemma-2-UD-Q6_K_XL.gguf", + "format": "GGUF", + "file_size": 248812032, + "total_params": 271002648, + "architecture": "gemma-embedding2", + "quantization": "Q6_K", + "tensor_count": 413, + "metadata": { + "gemma-embedding2.attention.causal": "false", + "gemma-embedding2.attention.head_count": "4", + "gemma-embedding2.attention.head_count_kv": "[len=24]", + "gemma-embedding2.attention.key_length": "512", + "gemma-embedding2.attention.key_length_swa": "256", + "gemma-embedding2.attention.layer_norm_rms_epsilon": "0.000001", + "gemma-embedding2.attention.shared_kv_layers": "0", + "gemma-embedding2.attention.sliding_window": "1024", + "gemma-embedding2.attention.sliding_window_pattern": "[len=0]", + "gemma-embedding2.attention.value_length": "512", + "gemma-embedding2.attention.value_length_swa": "256", + "gemma-embedding2.block_count": "24", + "gemma-embedding2.context_length": "262144", + "gemma-embedding2.embedding_length": "512", + "gemma-embedding2.embedding_length_out": "768", + "gemma-embedding2.embedding_length_per_layer_input": "512", + "gemma-embedding2.feed_forward_length": "2048", + "gemma-embedding2.pooling_type": "1", + "gemma-embedding2.rope.dimension_count": "512", + "gemma-embedding2.rope.dimension_count_swa": "256", + "gemma-embedding2.rope.freq_base": "1000000", + "gemma-embedding2.rope.freq_base_swa": "10000", + "general.architecture": "gemma-embedding2", + "general.basename": "embeddinggemma", + "general.file_type": "18", + "general.languages": "[len=1]", + "general.license": "apache-2.0", + "general.name": "Embeddinggemma 2", + "general.quantization_version": "2", + "general.size_label": "271M", + "general.tags": "[len=13]", + "general.type": "model", + "general.version": "2", + "quantize.imatrix.chunks_count": "230", + "quantize.imatrix.dataset": "data/calib.txt", + "quantize.imatrix.entries_count": "216", + "quantize.imatrix.file": "outputs/eg2/gguf/work/imatrix.gguf", + "tokenizer.chat_template": "{%- for msg in messages if msg.get('role') == 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- for item in msg['content'] if item.get('type') == 'text' -%}{{ item['text'] }}{%- endfor -%}{%- endif -%}{%- endfor -%}{%- for msg in messages if msg.get('role') != 'system' -%}{%- if msg.get('content') is string -%}{{ msg['content'] }}{%- else -%}{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}{%- for item in msg['content'] -%}{%- if item.get('type') == 'text' -%}{{ item['text'] }}{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}<|image|>{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}<|video|>{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}<|audio|>{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}", + "tokenizer.ggml.add_bos_token": "true", + "tokenizer.ggml.add_eos_token": "true", + "tokenizer.ggml.add_sep_token": "false", + "tokenizer.ggml.add_space_prefix": "false", + "tokenizer.ggml.bos_token_id": "2", + "tokenizer.ggml.eos_token_id": "1", + "tokenizer.ggml.mask_token_id": "4", + "tokenizer.ggml.merges": "[len=514906]", + "tokenizer.ggml.model": "gemma4", + "tokenizer.ggml.padding_token_id": "0", + "tokenizer.ggml.scores": "[len=262144]", + "tokenizer.ggml.token_type": "[len=262144]", + "tokenizer.ggml.tokens": "[len=262144]", + "tokenizer.ggml.unknown_token_id": "3" + } +} diff --git a/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.tensors.json b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.tensors.json new file mode 100644 index 0000000000..906782060d --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/b/unsloth-embeddinggemma-2-UD-Q6_K_XL.tensors.json @@ -0,0 +1,3531 @@ +{ + "file": "embeddinggemma-2-UD-Q6_K_XL.gguf", + "format_version": "GGUF v3", + "tensor_count": 413, + "total_size_bytes": 233020512, + "tensors": [ + { + "name": "output.weight", + "shape": [ + 512, + 768 + ], + "dtype": "Q6_K", + "size_bytes": 322560 + }, + { + "name": "output_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "per_layer_model_proj.weight", + "shape": [ + 512, + 12288 + ], + "dtype": "BF16", + "size_bytes": 12582912 + }, + { + "name": "per_layer_proj_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "token_embd.weight", + "shape": [ + 512, + 262144 + ], + "dtype": "Q6_K", + "size_bytes": 110100480 + }, + { + "name": "blk.0.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.0.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.0.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.0.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.0.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.0.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.0.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.0.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.0.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.0.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.0.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.0.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.1.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.1.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.1.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.1.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.1.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.1.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.1.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.1.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.1.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.1.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.1.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.2.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.2.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.2.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.2.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.2.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.2.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.2.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.2.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.2.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.2.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.2.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.3.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.3.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.3.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.3.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.3.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.3.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.3.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.3.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.3.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.3.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.3.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.4.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.4.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.4.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.4.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.4.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.4.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.4.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.4.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.4.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.4.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.4.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.5.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.5.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.5.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.5.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.5.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.5.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.6.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.6.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.6.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.6.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.6.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.6.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.6.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.6.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.6.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.6.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.6.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.7.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.7.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.7.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.7.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.7.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.7.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.7.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.7.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.7.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.7.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.7.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.8.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.8.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.8.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.8.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.8.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.8.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.8.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.8.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.8.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.8.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.8.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.9.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.9.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.9.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.9.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.9.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.9.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.9.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.9.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.9.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.9.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.9.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.10.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.10.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.10.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.10.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.10.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.10.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.10.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.10.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.10.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.10.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.10.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.11.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.11.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.11.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.11.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.11.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.11.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.12.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.12.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.12.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.12.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.12.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.12.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.12.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.12.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.12.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.12.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.12.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.13.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.13.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.13.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.13.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.13.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.13.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.13.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.13.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.13.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.13.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.13.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.14.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.14.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.14.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.14.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.14.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.14.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.14.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.14.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.14.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.14.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.14.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.15.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.15.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.15.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.15.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.15.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.15.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.15.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.15.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.15.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.15.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.15.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.16.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.16.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.16.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.16.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.16.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.16.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.16.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.16.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.16.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.16.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.16.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.17.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.17.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.17.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.17.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.17.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.17.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.18.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.18.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.18.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.18.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.18.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.18.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.18.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.18.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.18.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.18.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.18.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.19.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.19.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.19.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.19.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.19.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.19.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.19.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.19.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.19.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.19.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.19.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.20.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.20.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.20.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.20.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.20.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.20.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.20.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.20.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.20.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.20.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.20.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.21.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.21.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.21.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.21.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.21.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.21.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.21.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.21.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.21.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.21.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.21.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.22.attn_k_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.attn_output.weight", + "shape": [ + 1024, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.22.attn_q.weight", + "shape": [ + 512, + 1024 + ], + "dtype": "Q6_K", + "size_bytes": 430080 + }, + { + "name": "blk.22.attn_q_norm.weight", + "shape": [ + 256 + ], + "dtype": "F32", + "size_bytes": 1024 + }, + { + "name": "blk.22.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.22.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.22.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.22.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.22.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.22.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.22.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.22.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.attn_k.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.23.attn_k_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_output.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.attn_q.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.attn_q_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.attn_v.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 215040 + }, + { + "name": "blk.23.ffn_down.weight", + "shape": [ + 2048, + 512 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.ffn_gate.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.ffn_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.ffn_up.weight", + "shape": [ + 512, + 2048 + ], + "dtype": "Q6_K", + "size_bytes": 860160 + }, + { + "name": "blk.23.inp_gate.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + }, + { + "name": "blk.23.layer_output_scale.weight", + "shape": [ + 1 + ], + "dtype": "F32", + "size_bytes": 4 + }, + { + "name": "blk.23.post_attention_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_ffw_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.post_norm.weight", + "shape": [ + 512 + ], + "dtype": "F32", + "size_bytes": 2048 + }, + { + "name": "blk.23.proj.weight", + "shape": [ + 512, + 512 + ], + "dtype": "Q8_0", + "size_bytes": 278528 + } + ] +} diff --git a/docs/audits/APR-EMBED-001/EG-0/d-totality-mutation.md b/docs/audits/APR-EMBED-001/EG-0/d-totality-mutation.md new file mode 100644 index 0000000000..0e7398a75e --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/d-totality-mutation.md @@ -0,0 +1,23 @@ +# EG-0 (d) — kind-totality guard: first green and mutation RED + +Target: `cargo test -p apr-cli --test cli_commands` (an existing required target; no new job). +Binary tree: f745e7abc + this branch's registry and test edits. Run on the CPU test host, 2026-10-07. + +| Run | Contract state | `every_command_declares_one_verdict_per_model_kind` | +|---|---|---| +| green | as committed | ok (17/17 in the target) | +| mutation 1 | `run`/`chat`/`code` embedding verdict `refuse` → `advisory` | FAILED: `run: kind embedding has verdict advisory, not one of ["receipt", "refuse", "no-model"]` | +| mutation 2 | first `kinds:` line deleted (`validate-manifest`) | FAILED: `validate-manifest: no kinds: map` | +| restore | as committed | ok | + +`kind_totality_guard_case_table` carries 12 planted must-fail cases (zero commands, no list, +missing map, missing kind, fourth state, third kind, model verb says no-model, refusal without +`use_instead`, `use_instead` naming an unregistered verb, naming a verb that refuses the same +kind, `use_instead` on a receipt, no-model for one kind only) and one must-pass registry. + +## Re-run after the complexity split + +The guard was split into five functions (max cyclomatic 6) to meet the pre-commit +complexity gate. Mutation 1 re-run on the split code: RED, ``run: kind `embedding` has +verdict `advisory`, not one of ["receipt", "refuse", "no-model"]``; restore GREEN. All 17 +tests in the target pass. diff --git a/docs/audits/APR-EMBED-001/EG-0/m9.md b/docs/audits/APR-EMBED-001/EG-0/m9.md new file mode 100644 index 0000000000..7ad3ccf213 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-0/m9.md @@ -0,0 +1,88 @@ +# APR-EMBED-001 EG-0 (b) and (e) — M9 resolved from the files + +Binary: `apr 0.70.1 (f745e7abc)`, pinned with `scripts/apr_bin.sh`, run on the CPU +test host. No GPU was used. Raw output for every file is in `b/`: `.inspect.json` +from `"$APR" inspect --json` and `.tensors.json` from +`"$APR" tensors --json`. All 16 runs exited 0. + +## Exact file names (6 of 6) + +| Repo | File | +|------|------| +| unsloth/embeddinggemma-2-GGUF | `embeddinggemma-2-UD-Q4_K_XL.gguf`, `embeddinggemma-2-UD-Q5_K_XL.gguf`, `embeddinggemma-2-UD-Q6_K_XL.gguf`, `embeddinggemma-2-Q8_0.gguf`, `embeddinggemma-2-BF16.gguf`, `embeddinggemma-2-F16.gguf` | +| ggml-org/embeddinggemma-2-GGUF | `embeddinggemma-2-Q8_0.gguf`, `embeddinggemma-2-BF16.gguf` | + +## M9 items + +| Item | Value | Command | +|------|-------|---------| +| Architecture string | `gemma-embedding2` | `jq -r '.metadata["general.architecture"]' b/.inspect.json` | +| Causal flag | `attention.causal = false` | `jq '.metadata["gemma-embedding2.attention.causal"]'` | +| Pooling type | `pooling_type = 1` (mean) | `jq '.metadata["gemma-embedding2.pooling_type"]'` | +| Window | `attention.sliding_window = 1024`; `sliding_window_pattern` is an **empty array** | `jq '.metadata \| with_entries(select(.key \| test("sliding")))'` | +| Per-layer KV heads | `head_count_kv` is a 24-long array whose **values no apr verb prints** (finding below). From shapes: `attn_k` is `[512,512]` in every block, so KV heads = 1 in global blocks (head dim 512) and 2 in local blocks (head dim 256). Q heads = 4 in both | `jq '.tensors[] \| select(.name \| test("attn_(q\|k)(_norm)?.weight")) \| [.name,.shape]' b/.tensors.json` | +| Global / local layers | Global = blocks 5, 11, 17, 23 (`attn_q [512,2048]`, `attn_q_norm [512]`); local = the other 20 (`attn_q [512,1024]`, `attn_q_norm [256]`). Read from shapes, not from an index pattern (I-9) | same as above | +| RoPE | Two bases: `rope.freq_base = 1000000` (global), `rope.freq_base_swa = 10000` (local); `rope.dimension_count = 512`, `rope.dimension_count_swa = 256`. No `rope_freqs` tensor | `jq '.metadata \| with_entries(select(.key \| test("rope")))'` | +| Projection 512→768 in the main GGUF | **Yes**: `output.weight [512,768]`; `embedding_length = 512`, `embedding_length_out = 768` | `jq '.tensors[] \| select(.name=="output.weight")'` | +| Post-norms | Yes: `post_attention_norm`, `post_ffw_norm`, `post_norm` in every block | tensor name census below | +| QK-norm | Yes: `attn_q_norm`, `attn_k_norm` in every block | tensor name census below | +| Per-layer input | `inp_gate`, `proj` per block; `per_layer_model_proj`, `per_layer_proj_norm` global; `embedding_length_per_layer_input = 512`. No per-layer token embedding table | tensor name census below | +| Layer output scale | `layer_output_scale` per block | tensor name census below | +| Softcap | No softcap metadata key in any file | `jq '.metadata \| keys[] \| select(test("softcap"))'` (empty) | +| Shared KV layers | `attention.shared_kv_layers = 0` | `jq '.metadata["gemma-embedding2.attention.shared_kv_layers"]'` | +| F16 degrades on the oracle | **pending (c)**: needs the oracle build at or after llama.cpp PR 30054 | — | + +Tensor name census (identical layout in all 8 files): +`jq -r '.tensors[].name' b/.tensors.json | sed -E 's/blk\.[0-9]+\./blk.N./' | sort | uniq -c` +gives 17 names × 24 blocks (`attn_{q,k,v,output}`, `attn_{q,k}_norm`, `attn_norm`, +`ffn_{gate,up,down}`, `ffn_norm`, `inp_gate`, `proj`, `layer_output_scale`, +`post_attention_norm`, `post_ffw_norm`, `post_norm`) plus 5 global +(`token_embd`, `output`, `output_norm`, `per_layer_model_proj`, `per_layer_proj_norm`). + +## Per-tensor quant types in the three UD files + +Command: `jq -r '.tensors[] | [.name, .dtype] | @tsv' b/.tensors.json | sed -E 's/blk\.[0-9]+\./blk.N./' | sort | uniq -c` + +| Tensor | UD-Q4_K_XL | UD-Q5_K_XL | UD-Q6_K_XL | +|--------|-----------|-----------|-----------| +| block matrices (attn_q/k/output, ffn_gate/up) | Q4_K | Q5_K | Q6_K | +| attn_v, ffn_down | Q4_K | 12 × Q5_K, 12 × Q6_K | Q6_K | +| inp_gate, proj | Q8_0 | Q8_0 | Q8_0 | +| token_embd | Q4_K | Q5_K | Q6_K | +| output (projection) | Q6_K | Q6_K | Q6_K | +| per_layer_model_proj | Q5_K | Q6_K | BF16 | +| norms, layer_output_scale | F32 | F32 | F32 | + +## Findings the spec did not predict + +1. **Both repos ship `mmproj` (vision and audio projector) files.** The spec's + M3 / D-4 premise is text-only weights. The text GGUFs above carry no vision + tensors; the projectors are separate files. I-7 (media refused) still holds. +2. **`context_length` metadata is 262144**; the model card says 8192. The + truncation contract must take the card's limit, or the oracle's, not the key. +3. **`tokenizer.ggml.model = gemma4`**, not the SentencePiece model EG-1's text names. The + vocabulary is still 262,144. +4. **`head_count_kv` per-layer values are invisible**: `apr inspect` prints the + array as `[len=24]`. EG-1 needs the values; the shapes above give them meanwhile. +5. `apr flow` labels the file "decoder-only (LLaMA/Qwen2)" and `apr tokenize` + falls back to greedy with `roundtrip: false`. Both exit 0, so both are silent + wrong answers (probe receipt `a-probe-udq4.tsv`). + +## (e) Capability contract + +`contracts/apr-model-capability-v1.yaml` (mirror byte-identical) gains eight ops, +each `gpu_supported: false` with a reason: `BidirectionalMask`, `SlidingWindowAttn`, +`GatedGeluMlp`, `PerLayerInput`, `LayerOutputScale`, `MeanPool`, `DenseProjection`, +`L2Normalize`. `op_derivation.always` no longer holds `CausalMask`; `by_kind` +gives `generative: [CausalMask]` and `embedding: [BidirectionalMask, MeanPool, +DenseProjection, L2Normalize]`, and `per_file` derives the rest from tensors and +metadata. `RequiredOp` gains the eight variants (21 in all); `scripts/capability_op_impl_map.txt` gains +eight `-` rows. Nothing derives the new ops yet (EG-1/EG-2). + +| Check | Result | +|-------|--------| +| `cargo test -p aprender-serve --lib capability` | 31 passed | +| `cargo test -p apr-cli --test capability_mirror` | 3 passed | +| `bash scripts/check_unwired_capabilities.sh` | exit 0 | +| Mutation: delete the `MeanPool` row | RED, `RequiredOp::MeanPool is not declared in the contract` | +| Restore | GREEN, 6 passed | diff --git a/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv b/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv index fe0cc74724..2317d0a175 100644 --- a/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv +++ b/docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv @@ -1,5 +1,5 @@ # APR-EMBED-001 EG-0(a): one probe per verb. Columns: verbargs. -# {M}=model under test {B}=BF16 sibling {O}=fresh scratch dir. Every probe runs with +# {M}=model under test {B}=BF16 sibling {D}=the model's directory {O}=fresh scratch dir. Every probe runs with # stdin 72 h it is a §8 andon with the ticket named. Work on §1.5 diff --git a/scripts/apr_embed_probe.sh b/scripts/apr_embed_probe.sh index f6fa573e94..0dc2c8f355 100755 --- a/scripts/apr_embed_probe.sh +++ b/scripts/apr_embed_probe.sh @@ -15,6 +15,8 @@ TIMEOUT_S="${APR_EMBED_PROBE_TIMEOUT:-120}" . scripts/apr_bin.sh || exit 1 +LOGS="${OUT%.tsv}.logs" +mkdir -p "${LOGS}" SCRATCH="$(mktemp -d)" trap 'rm -rf "${SCRATCH}"' EXIT @@ -29,12 +31,13 @@ while IFS=$'\t' read -r verb args; do mkdir -p "${o}" args="${args//\{M\}/${MODEL}}" args="${args//\{B\}/${BF16}}" + args="${args//\{D\}/$(dirname "${MODEL}")}" args="${args//\{O\}/${o}}" # Word-splitting of the substituted args is intended: the table holds argv. # shellcheck disable=SC2086 rc=0 - timeout "${TIMEOUT_S}" "${APR}" "${verb}" ${args} < /dev/null > "${o}.log" 2>&1 || rc=$? - first="$(grep -m1 -vE '^[[:space:]]*$|^[━─╭╰│]+$' "${o}.log" | tr '\t' ' ' | cut -c1-200 || true)" + timeout "${TIMEOUT_S}" "${APR}" "${verb}" ${args} < /dev/null > "${LOGS}/${verb}.log" 2>&1 || rc=$? + first="$(grep -m1 -vE '^[[:space:]]*$|^[━─╭╰│]+$' "${LOGS}/${verb}.log" | tr '\t' ' ' | cut -c1-200 || true)" printf '%s\t%s\t%s\n' "${verb}" "${rc}" "${first}" >> "${OUT}" n=$((n + 1)) done < "${TABLE}" diff --git a/scripts/capability_op_impl_map.txt b/scripts/capability_op_impl_map.txt index 36c39eebba..8600b21016 100644 --- a/scripts/capability_op_impl_map.txt +++ b/scripts/capability_op_impl_map.txt @@ -36,3 +36,11 @@ LayerNorm layer_norm_gpu UNSUPPORTED, but crates/aprender-serve/src/cuda/executo AbsolutePos - UNSUPPORTED and unimplemented; GPU uses RoPE AttnFinalSoftcap - UNSUPPORTED and unimplemented; PMAT-824 PostAttnFfnNorm - UNSUPPORTED and unimplemented; PMAT-824 +BidirectionalMask - UNSUPPORTED and unimplemented; APR-EMBED-001 (embedding kind) +SlidingWindowAttn - UNSUPPORTED and unimplemented; APR-EMBED-001 +GatedGeluMlp - UNSUPPORTED and unimplemented; APR-EMBED-001. Not gelu_gpu (activation kernel) and not fused_swiglu_gpu (SiLU gate) +PerLayerInput - UNSUPPORTED and unimplemented; APR-EMBED-001 +LayerOutputScale - UNSUPPORTED and unimplemented; APR-EMBED-001 +MeanPool - UNSUPPORTED and unimplemented; APR-EMBED-001 +DenseProjection - UNSUPPORTED and unimplemented; APR-EMBED-001 +L2Normalize - UNSUPPORTED and unimplemented; APR-EMBED-001 From b78619a14dc583d8ec3f7a805eac03633166542b Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 01:29:30 +0200 Subject: [PATCH 04/21] fix(embed): EG-0 forward contract passes the ontology gates The new embeddinggemma2-forward-v1 contract tripped six tests in workspace-test: its ten `formal:` lines carried no declared symbol (the shrink-only formal_prose baseline rose 1464 -> 1473), its lean_theorem labels named theorems that do not exist, and contracts.nt plus the witness were not regenerated. - Rewrite every `formal:` with symbols the ontology declares. - Drop the lean_theorem labels; no Lean proof exists yet. - Regenerate contracts.nt (pv extract) and the witness (pv-sat). Verified: ont2b_sigma_gate 8, ont4b_shapes_gate 11, ont5_consistency_gate 5, ont8_evidence_gate 8, pvl_discharge_check 15, all passing. No baseline or allow-list was edited. Refs #4899 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/contracts.nt | 5 ++++ contracts/embeddinggemma2-forward-v1.yaml | 28 +++++++------------ ...1a9726ef28aaee70bad35dff472cef083bf67.json | 6 ++-- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 93c2f8bbab..a0d212aaf9 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -11451,6 +11451,11 @@ . "contracts/embedding-lookup-v1.yaml"^^ . "embedding-lookup-v1"^^ . + . + . + . + "contracts/embeddinggemma2-forward-v1.yaml"^^ . + "embeddinggemma2-forward-v1"^^ . . . . diff --git a/contracts/embeddinggemma2-forward-v1.yaml b/contracts/embeddinggemma2-forward-v1.yaml index 50f22b1cdc..fd9a228090 100644 --- a/contracts/embeddinggemma2-forward-v1.yaml +++ b/contracts/embeddinggemma2-forward-v1.yaml @@ -23,7 +23,6 @@ equations: preconditions: - block_count > 0 - every block has an attn_q tensor - lean_theorem: Theorems.EmbeddingGemma2_Layer_Kind postconditions: - 'every layer has exactly one kind and the kind set is {local, global}' local_attention: @@ -36,7 +35,6 @@ equations: preconditions: - W > 0 - n > 0 - lean_theorem: Theorems.EmbeddingGemma2_Local_Attention postconditions: - 'output.shape == input.shape and all output values are finite' global_attention: @@ -48,7 +46,6 @@ equations: - Padding positions receive zero weight preconditions: - n > 0 - lean_theorem: Theorems.EmbeddingGemma2_Global_Attention postconditions: - 'output.shape == input.shape and all output values are finite' gated_gelu_ffn: @@ -59,7 +56,6 @@ equations: - The activation is GELU (tanh approximation), never SiLU; a gated MLP does not imply SwiGLU preconditions: - ffn_gate, ffn_up and ffn_down are present in every block - lean_theorem: Theorems.EmbeddingGemma2_Gated_Gelu_Ffn postconditions: - 'output.shape == input.shape and all output values are finite' mean_pool: @@ -72,7 +68,6 @@ equations: preconditions: - pooling_type read from metadata is mean - '|R| >= 1' - lean_theorem: Theorems.EmbeddingGemma2_Mean_Pool postconditions: - 'output.len() == d_model' projection: @@ -84,7 +79,6 @@ equations: - A file without the projection tensor is refused by name, never served at trunk width preconditions: - output.weight exists with shape [d_model, d_out] - lean_theorem: Theorems.EmbeddingGemma2_Projection postconditions: - 'output.len() == embedding_length_out' l2_normalise: @@ -96,7 +90,6 @@ equations: - Every component is finite (I-5) preconditions: - '||e||_2 > 0 and finite' - lean_theorem: Theorems.EmbeddingGemma2_L2_Normalise postconditions: - 'output.len() == embedding_length_out' truncation: @@ -108,59 +101,58 @@ equations: - t_k for k = d_model (512) is not the pooled trunk state preconditions: - k is one of the declared Matryoshka dimensions - lean_theorem: Theorems.EmbeddingGemma2_Truncation postconditions: - 'output.len() == k and abs(norm(output) - 1) <= 1e-5' proof_obligations: - id: EG2-INV-001 type: invariant property: Layer kind from the file - formal: kind(l) is a function of tensor shapes and metadata only; renumbering layers in a synthetic file moves the kinds with the tensors + formal: "∀ l ∈ blocks: kind(l) = f(shape(attn_q[l]), metadata) ∧ kind(l) ∈ {local, global}" applies_to: all - id: EG2-INV-002 type: invariant property: Local attention window - formal: weight(i, j) == 0 for |i - j| >= W in every local layer + formal: "∀ l ∈ local, ∀ i, j: abs(i - j) ≥ W ⇒ weight(i, j) = 0" applies_to: all - id: EG2-INV-003 type: invariant property: Bidirectional global attention - formal: for real tokens i, j in a global layer, weight(i, j) > 0 and weight(j, i) > 0 + formal: "∀ l ∈ global, ∀ i, j ∈ real: weight(i, j) > 0 ∧ weight(j, i) > 0" applies_to: all - id: EG2-INV-004 type: invariant property: Gated GELU activation - formal: FFN uses GELU_tanh on the gate branch; substituting SiLU changes the output beyond tolerance + formal: "FFN(h) = W_down · (gelu_tanh(W_gate · h) × (W_up · h)) ∧ FFN_silu(h) ≠ FFN(h)" applies_to: all - id: EG2-INV-005 type: invariant property: Mean pool over real tokens only - formal: appending padding to a sequence leaves p unchanged within 1e-6 + formal: "∀ pad: ‖pool(x ++ pad) - pool(x)‖ ≤ 1e-6" applies_to: all - id: EG2-INV-006 type: invariant property: Projected output - formal: len(e) == embedding_length_out and e == W_out p + formal: "len(e) = embedding_length_out ∧ e = W_out · p" applies_to: all - id: EG2-BND-007 type: bound property: Unit norm and finite - formal: '| ||u||_2 - 1 | <= 1e-5 and is_finite(u[i]) for all i' + formal: "abs(‖u‖₂ - 1) ≤ 1e-5 ∧ ∀ i: isFinite(u[i])" applies_to: all - id: EG2-INV-008 type: invariant property: Truncation re-normalises the leading dims - formal: t_k == normalise(u_full[0..k]) for each k in {768, 512, 256, 128} + formal: "∀ k ∈ {768, 512, 256, 128}: t_k = u[0..k] / ‖u[0..k]‖₂" applies_to: all - id: EG2-INV-009 type: invariant property: Batch and order invariance - formal: embed(x) == embed_batch([.., x, ..])[pos] within 1e-5 for every batch size and position + formal: "∀ b ∈ {1, 8, 32}, ∀ pos < b: ‖embed(x) - embed_batch(B)[pos]‖ ≤ 1e-5" applies_to: all - id: EG2-EQ-010 type: equivalence property: Oracle parity - formal: cosine(apr(x), oracle(x)) >= threshold for every corpus item, threshold from the thresholds file's known-good and known-bad method + formal: "∀ x ∈ corpus: cosine(apr(x), oracle(x)) ≥ threshold" applies_to: all falsification_tests: - id: FALSIFY-EG2-001 diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 644709151a..96e03595d0 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "1d974bf825181bf6340c1b15622b5f0adf9c9d7e9728b137bcce9f0d13e5db5d", + "census_id_set_sha256": "10022e507347c1a63f05fe960f1daefa699661aab6d1fcd80e16ba491d0c8064", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "82da4e29ad0c1d9d123f491939a5bb982871be4d", + "reasoner_git_sha": "3207b354e35633c2c5fba85876adb2091283bbd5", "checkable_n": 84, "result": { "kind": "model", @@ -10,5 +10,5 @@ } }, "pc_reasoner": "fired", - "cpu_ms": 6 + "cpu_ms": 0 } From e143f894d03443896c2eb5ea283de549fc15e3dc Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 02:52:57 +0200 Subject: [PATCH 05/21] =?UTF-8?q?feat(embed):=20EG-1=20slice=201=20?= =?UTF-8?q?=E2=80=94=20ModelKind,=20per-layer=20KV=20heads=20and=20layer?= =?UTF-8?q?=20kind=20from=20the=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - model_kind.rs: ModelKind {Generative, Embedding} by an exact, case-insensitive match on gemma-embedding2; the gemma prefix match is not widened. A causal embedding file is refused. per_layer_kv_heads reads the head_count_kv array per layer (scalar broadcasts, wrong length refused); layer_attention_kinds reads local/global from attn_q widths (I-9). - contract_gate: gemma-embedding2 is refused on the generative path at gate model_kind, naming `apr embed` (I-3), instead of the Gemma3 message. - contracts/embeddinggemma2-load-v1.yaml: 5 obligations, 5 falsifiers. FALSIFY-EG2L-004 observed RED before the gate change, GREEN after. Five planted mutants in model_kind.rs (prefix match, causal guard, first-entry KV, kind flip, length check) each turn a test RED. aprender-serve lib: model_kind + contract_gate 50 passed; clippy -D warnings clean; ont2b/ont4b/ont5/ont8/pvl_discharge green. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/contracts.nt | 5 + contracts/embeddinggemma2-load-v1.yaml | 114 ++++++++++++++ ...1a9726ef28aaee70bad35dff472cef083bf67.json | 2 +- crates/aprender-serve/src/contract_gate.rs | 35 +++++ crates/aprender-serve/src/lib.rs | 3 + crates/aprender-serve/src/model_kind.rs | 139 +++++++++++++++++ crates/aprender-serve/src/model_kind_tests.rs | 143 ++++++++++++++++++ 7 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 contracts/embeddinggemma2-load-v1.yaml create mode 100644 crates/aprender-serve/src/model_kind.rs create mode 100644 crates/aprender-serve/src/model_kind_tests.rs diff --git a/contracts/contracts.nt b/contracts/contracts.nt index a0d212aaf9..ba05737548 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -11456,6 +11456,11 @@ . "contracts/embeddinggemma2-forward-v1.yaml"^^ . "embeddinggemma2-forward-v1"^^ . + . + . + "contracts/embeddinggemma2-load-v1.yaml"^^ . + "embeddinggemma2-load-v1"^^ . + "schema"^^ . . . . diff --git a/contracts/embeddinggemma2-load-v1.yaml b/contracts/embeddinggemma2-load-v1.yaml new file mode 100644 index 0000000000..e7518c51a4 --- /dev/null +++ b/contracts/embeddinggemma2-load-v1.yaml @@ -0,0 +1,114 @@ +metadata: + version: 1.0.0 + kind: schema + created: '2026-10-08' + author: PAIML Engineering + description: EmbeddingGemma 2 load -- model kind by exact architecture match, per-layer KV heads and attention kind from the file (APR-EMBED-001 EG-1) + references: + - docs/specifications/APR-EMBED-001-embeddinggemma-2-support.md (I-3, I-9) + - docs/audits/APR-EMBED-001/EG-0/m9.md (architecture string, causal flag, per-layer shapes) + depends_on: + - embeddinggemma2-forward-v1 +equations: + model_kind: + formula: kind(f) = embedding if arch(f) equals gemma-embedding2 (ASCII case-insensitive), else generative + domain: f a GGUF file with general.architecture and optional attention.causal + codomain: '{generative, embedding}' + invariants: + - The match is exact; the gemma prefix match that refuses Gemma3 is not widened + - An embedding architecture with attention.causal = true is refused, never served under either kind + preconditions: + - general.architecture is present + postconditions: + - 'every admitted file has exactly one kind' + generative_path_refusal: + formula: load_generative(f) = Refused(model_kind, names apr embed) if kind(f) = embedding + domain: f with kind(f) = embedding + codomain: a refusal naming the verb that serves the file + invariants: + - The refusal names the kind and the verb (I-3); it never names Gemma3 or its ticket + preconditions: + - kind(f) = embedding + postconditions: + - 'no generative forward pass runs on an embedding file' + per_layer_kv_heads: + formula: kv(l) = head_count_kv[l] if head_count_kv is an array, else head_count_kv for every l + domain: l in [0, block_count), head_count_kv read from the GGUF + codomain: positive integers + invariants: + - Read from the file, never from layer-index arithmetic and never collapsed to the query head count (I-9) + - An array whose length differs from block_count is refused + preconditions: + - block_count > 0 + postconditions: + - 'len(kv) == block_count and every entry is > 0' + layer_attention_kind: + formula: attn(l) = global if width(attn_q[l]) equals the widest attn_q in the file, else local + domain: l in [0, block_count), attn_q output widths read from tensor shapes + codomain: '{local, global}' + invariants: + - Swapping the shapes of two layers swaps their kinds (I-9) + - A file with a single attn_q width is refused; this architecture has both kinds + preconditions: + - every block has an attn_q tensor of non-zero width + postconditions: + - 'at least one layer is global and at least one is local' +proof_obligations: +- id: EG2L-INV-001 + type: invariant + property: Exact architecture match + formal: "∀ a: kind(a) = embedding ↔ lower(a) = gemma-embedding2" + applies_to: all +- id: EG2L-INV-002 + type: invariant + property: Causal contradiction refused + formal: "∀ f: kind(f) = embedding ∧ causal(f) ⇒ refused(f)" + applies_to: all +- id: EG2L-INV-003 + type: invariant + property: Per-layer KV heads from the file + formal: "∀ l < block_count: kv(l) = head_count_kv[l] ∧ kv(l) ≥ 1 ∧ len(kv) = block_count" + applies_to: all +- id: EG2L-INV-004 + type: invariant + property: Layer attention kind from shapes + formal: "∀ l: attn(l) = global ↔ width(attn_q[l]) = max(width(attn_q))" + applies_to: all +- id: EG2L-INV-005 + type: invariant + property: Generative path refuses by kind + formal: "∀ f: kind(f) = embedding ⇒ load_generative(f) = refused(model_kind)" + applies_to: all +falsification_tests: +- id: FALSIFY-EG2L-001 + rule: Exact architecture match + prediction: gemma-embedding2 in any case is embedding; gemma, gemma2, gemma3, gemma-embedding, gemma-embedding3 and gemma-embedding2x are generative + test: model_kind::tests::falsify_eg2l_001_exact_arch_match_only + if_fails: The match was widened to a prefix +- id: FALSIFY-EG2L-002 + rule: Per-layer KV heads from the file + prediction: A 24-long array yields 24 per-layer counts in file order; a scalar broadcasts; a wrong length, zero, negative or non-integer entry is refused + test: model_kind::tests::falsify_eg2l_002_* + if_fails: The array collapsed to the query head count or to its first entry (the num_kv_heads UInt32-only reader) +- id: FALSIFY-EG2L-003 + rule: Layer attention kind from shapes + prediction: Globals are exactly the widest attn_q layers; swapping shapes swaps kinds; one width is refused + test: model_kind::tests::falsify_eg2l_003_* + if_fails: Layer kind derived from an index pattern or an architecture constant +- id: FALSIFY-EG2L-004 + rule: Generative path refuses by kind + prediction: validate_model_load on gemma-embedding2 fails at gate model_kind naming apr embed, not Gemma3; gemma-embedding3 still reaches the Gemma3 refusal + test: contract_gate::tests::falsify_eg2l_004_embedding_refused_by_kind_on_generative_path + if_fails: The file was refused with the Gemma3 message, or loaded as a generative model +- id: FALSIFY-EG2L-005 + rule: Causal contradiction refused + prediction: gemma-embedding2 with attention.causal = true is refused naming the flag; false or absent is embedding + test: model_kind::tests::falsify_eg2l_001_kind_and_causal_flag + if_fails: A causal embedding file was admitted under either kind +qa_gate: + id: F-EG2L-001 + name: embeddinggemma2-load-v1 Contract + description: Quality gate for loading EmbeddingGemma 2 by kind + checks: + - validation + - falsification diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 96e03595d0..e55959c2ae 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,5 +1,5 @@ { - "census_id_set_sha256": "10022e507347c1a63f05fe960f1daefa699661aab6d1fcd80e16ba491d0c8064", + "census_id_set_sha256": "1974c0d8adbfcf354490c7138c476f9215be185cfff9e8bddd606fdd988af1fa", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", "reasoner_git_sha": "3207b354e35633c2c5fba85876adb2091283bbd5", "checkable_n": 84, diff --git a/crates/aprender-serve/src/contract_gate.rs b/crates/aprender-serve/src/contract_gate.rs index 7993ab6107..13ac4b3aa4 100644 --- a/crates/aprender-serve/src/contract_gate.rs +++ b/crates/aprender-serve/src/contract_gate.rs @@ -348,6 +348,17 @@ fn validate_supported_architecture(arch_name: &str) -> std::result::Result<(), M if is_gemma2_supported(arch_name) { return Ok(()); } + // APR-EMBED-001 EG-1: an embedding model is admitted by its own exact match, BEFORE the + // `gemma` prefix gate, and refused here by kind (I-3): it has no language-model head. + if crate::model_kind::is_embedding_gemma2(arch_name) { + return Err(ModelLoadError { + gate: "model_kind", + reason: format!( + "'{arch_name}' is an embedding model (ModelKind::Embedding): it has no \ + language-model head and does not generate text. Use `apr embed`." + ), + }); + } if is_gemma_family(arch_name) { return Err(ModelLoadError { gate: "architecture_supported", @@ -911,6 +922,30 @@ mod tests { } } + /// FALSIFY-EG2L-004 (APR-EMBED-001 EG-1): an EmbeddingGemma 2 file is refused on the + /// generative load path BY KIND, naming the verb that serves it (I-3) -- never with the + /// Gemma3 message, which named the wrong architecture and the wrong ticket. + #[test] + fn falsify_eg2l_004_embedding_refused_by_kind_on_generative_path() { + for name in ["gemma-embedding2", "Gemma-Embedding2"] { + let mut config = valid_config(); + config.architecture = name.to_string(); + let err = validate_model_load(&config).expect_err(&format!( + "embedding arch '{name}' must not load as generative" + )); + assert_eq!(err.gate, "model_kind", "'{name}': {}", err.reason); + assert!(err.reason.contains("apr embed"), "'{name}': {}", err.reason); + assert!(!err.reason.contains("Gemma3"), "'{name}': {}", err.reason); + } + // The gemma prefix match is not widened: near-misses still reach the Gemma3 refusal. + for name in ["gemma-embedding3", "gemma-embedding"] { + let mut config = valid_config(); + config.architecture = name.to_string(); + let err = validate_model_load(&config).expect_err(name); + assert_eq!(err.gate, "architecture_supported", "'{name}'"); + } + } + /// PMAT-810 FALSIFIER: Gemma v2 now LOADS (it was fail-loud under PMAT-807/809). /// /// If the forward path ever regresses and Gemma v2 is re-rejected, this fails. diff --git a/crates/aprender-serve/src/lib.rs b/crates/aprender-serve/src/lib.rs index 669aeb51ae..42690ac835 100644 --- a/crates/aprender-serve/src/lib.rs +++ b/crates/aprender-serve/src/lib.rs @@ -360,6 +360,9 @@ pub mod layers; pub mod memory; #[cfg(feature = "server")] pub mod metrics; +/// APR-EMBED-001 EG-1: model kind (generative or embedding) and the per-layer facts an embedding +/// model reads from its file (KV heads, local/global attention). +pub mod model_kind; /// Unified model loader for APR, GGUF, and SafeTensors /// /// Per spec §3.2 and §5: Combines format detection with model loading. diff --git a/crates/aprender-serve/src/model_kind.rs b/crates/aprender-serve/src/model_kind.rs new file mode 100644 index 0000000000..43ccb59775 --- /dev/null +++ b/crates/aprender-serve/src/model_kind.rs @@ -0,0 +1,139 @@ +//! APR-EMBED-001 EG-1: what kind of model a file is, and the per-layer facts an embedding model reads from it. +//! +//! The kind is decided by a distinct, EXACT architecture match. The `gemma` prefix match in +//! [`crate::contract_gate::is_gemma_family`] is not widened: `gemma-embedding2` is admitted here, before that +//! gate sees it, and every other `gemma*` string still reaches it unchanged. +//! +//! Per-layer facts come from the file (I-9): the KV-head count of each layer is read from the +//! `attention.head_count_kv` value, scalar or array, and the attention kind of each layer from the width of +//! its query projection. Never from layer-index arithmetic, never from an architecture constant. +//! +//! Contract: `contracts/embeddinggemma2-load-v1.yaml`. + +use crate::gguf::GGUFValue; + +/// The GGUF `general.architecture` string of EmbeddingGemma 2, as the files carry it. +pub const EMBEDDING_GEMMA2_ARCH: &str = "gemma-embedding2"; + +/// What a model file produces. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelKind { + /// Next-token logits from a causal language-model head. + Generative, + /// One pooled, projected vector per input (bidirectional attention, no language-model head). + Embedding, +} + +/// Is `arch` the EmbeddingGemma 2 architecture? Exact and case-insensitive; no prefix. +#[must_use] +pub fn is_embedding_gemma2(arch: &str) -> bool { + arch.eq_ignore_ascii_case(EMBEDDING_GEMMA2_ARCH) +} + +/// The kind of a file, from its architecture string and its `attention.causal` flag. +/// +/// An embedding architecture whose file says `causal = true` is a contradiction and is refused by name: it is +/// never served under either kind. +/// +/// # Errors +/// Returns the reason when the architecture and the causal flag disagree. +pub fn model_kind(arch: &str, causal: Option) -> Result { + if !is_embedding_gemma2(arch) { + return Ok(ModelKind::Generative); + } + match causal { + Some(true) => Err(format!( + "'{arch}' is an embedding architecture but the file sets attention.causal = true; \ + refusing a file whose metadata contradicts its architecture" + )), + Some(false) | None => Ok(ModelKind::Embedding), + } +} + +/// The KV-head count of every layer, from the `attention.head_count_kv` value. +/// +/// A scalar applies to every layer. An array gives one count per layer and must be exactly `block_count` long. +/// +/// # Errors +/// Returns the reason for an array of the wrong length, a zero or non-integer count, or any other value type. +pub fn per_layer_kv_heads(value: &GGUFValue, block_count: usize) -> Result, String> { + match value { + GGUFValue::Array(items) => { + if items.len() != block_count { + return Err(format!( + "attention.head_count_kv has {} entries for {block_count} blocks", + items.len() + )); + } + items.iter().map(kv_count).collect() + }, + scalar => Ok(vec![kv_count(scalar)?; block_count]), + } +} + +fn kv_count(value: &GGUFValue) -> Result { + let n = match value { + GGUFValue::UInt8(v) => u64::from(*v), + GGUFValue::UInt16(v) => u64::from(*v), + GGUFValue::UInt32(v) => u64::from(*v), + GGUFValue::UInt64(v) => *v, + GGUFValue::Int32(v) => { + u64::try_from(*v).map_err(|_| format!("negative KV-head count {v}"))? + }, + other => { + return Err(format!( + "attention.head_count_kv entry is not an integer: {other:?}" + )) + }, + }; + match usize::try_from(n) { + Ok(0) => Err("attention.head_count_kv entry is 0".to_string()), + Ok(k) => Ok(k), + Err(_) => Err(format!("KV-head count {n} does not fit usize")), + } +} + +/// The attention kind of one layer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LayerAttention { + /// Sliding-window attention over the declared window. + Local, + /// Full bidirectional attention over every real token. + Global, +} + +/// The attention kind of every layer, from the output width of each layer's query projection. +/// +/// A layer is global when its query projection is the widest in the file, local otherwise. A file whose +/// layers all have one width carries no local/global split and is refused: this architecture has both. +/// +/// # Errors +/// Returns the reason for an empty layer list, a zero width, or a file with a single width. +pub fn layer_attention_kinds(q_out_widths: &[usize]) -> Result, String> { + let (Some(&widest), Some(&narrowest)) = (q_out_widths.iter().max(), q_out_widths.iter().min()) + else { + return Err("no attn_q tensors: cannot read the layer attention kinds".to_string()); + }; + if narrowest == 0 { + return Err("an attn_q tensor has output width 0".to_string()); + } + if widest == narrowest { + return Err(format!( + "every attn_q has output width {widest}: the file carries no local/global split" + )); + } + Ok(q_out_widths + .iter() + .map(|&w| { + if w == widest { + LayerAttention::Global + } else { + LayerAttention::Local + } + }) + .collect()) +} + +#[cfg(test)] +#[path = "model_kind_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/model_kind_tests.rs b/crates/aprender-serve/src/model_kind_tests.rs new file mode 100644 index 0000000000..13a6202937 --- /dev/null +++ b/crates/aprender-serve/src/model_kind_tests.rs @@ -0,0 +1,143 @@ +//! Falsifiers for `contracts/embeddinggemma2-load-v1.yaml` (FALSIFY-EG2L-001..004). + +use super::*; + +// FALSIFY-EG2L-001: the kind comes from an exact architecture match, never from the `gemma` prefix. +#[test] +fn falsify_eg2l_001_exact_arch_match_only() { + assert!(is_embedding_gemma2("gemma-embedding2")); + assert!(is_embedding_gemma2("Gemma-Embedding2")); + for not_it in [ + "gemma", + "gemma2", + "gemma3", + "gemma-embedding", + "gemma-embedding3", + "gemma-embedding2x", + "embeddinggemma2", + "", + ] { + assert!( + !is_embedding_gemma2(not_it), + "{not_it} must not be admitted as EmbeddingGemma 2" + ); + assert_eq!( + model_kind(not_it, Some(false)), + Ok(ModelKind::Generative), + "{not_it}" + ); + } +} + +#[test] +fn falsify_eg2l_001_kind_and_causal_flag() { + assert_eq!( + model_kind("gemma-embedding2", Some(false)), + Ok(ModelKind::Embedding) + ); + assert_eq!( + model_kind("gemma-embedding2", None), + Ok(ModelKind::Embedding) + ); + let err = model_kind("gemma-embedding2", Some(true)) + .expect_err("causal embedding file must be refused"); + assert!(err.contains("attention.causal = true"), "{err}"); +} + +// FALSIFY-EG2L-002: per-layer KV heads are read from the array, not collapsed to num_heads. +#[test] +fn falsify_eg2l_002_kv_heads_array_read_per_layer() { + let layers: Vec = (0..24u32) + .map(|i| GGUFValue::UInt32(if i % 6 == 5 { 1 } else { 2 })) + .collect(); + let got = per_layer_kv_heads(&GGUFValue::Array(layers), 24).expect("24-long array"); + assert_eq!(got.len(), 24); + assert_eq!(got.iter().filter(|&&k| k == 1).count(), 4); + assert_eq!(got[5], 1); + assert_eq!(got[0], 2); +} + +#[test] +fn falsify_eg2l_002_kv_heads_follow_the_file_not_a_pattern() { + // Same counts, different positions: the reader must follow the file. + let vals = [3u32, 1, 4, 1, 5]; + let arr = GGUFValue::Array(vals.iter().map(|&v| GGUFValue::Int32(v as i32)).collect()); + assert_eq!(per_layer_kv_heads(&arr, 5), Ok(vec![3, 1, 4, 1, 5])); +} + +#[test] +fn falsify_eg2l_002_kv_heads_scalar_broadcasts() { + assert_eq!( + per_layer_kv_heads(&GGUFValue::UInt32(8), 3), + Ok(vec![8, 8, 8]) + ); + assert_eq!(per_layer_kv_heads(&GGUFValue::UInt64(2), 2), Ok(vec![2, 2])); + assert_eq!(per_layer_kv_heads(&GGUFValue::UInt8(1), 1), Ok(vec![1])); + assert_eq!(per_layer_kv_heads(&GGUFValue::UInt16(4), 1), Ok(vec![4])); +} + +#[test] +fn falsify_eg2l_002_kv_heads_refusals() { + let short = GGUFValue::Array(vec![GGUFValue::UInt32(1); 23]); + let err = per_layer_kv_heads(&short, 24).expect_err("length mismatch"); + assert!(err.contains("23 entries for 24 blocks"), "{err}"); + assert!( + per_layer_kv_heads(&GGUFValue::UInt32(0), 2).is_err(), + "zero heads" + ); + assert!( + per_layer_kv_heads(&GGUFValue::Int32(-1), 2).is_err(), + "negative heads" + ); + assert!( + per_layer_kv_heads(&GGUFValue::Float32(2.0), 2).is_err(), + "float heads" + ); + assert!( + per_layer_kv_heads(&GGUFValue::String("2".into()), 2).is_err(), + "string heads" + ); + let bad_entry = GGUFValue::Array(vec![GGUFValue::UInt32(1), GGUFValue::Bool(true)]); + assert!( + per_layer_kv_heads(&bad_entry, 2).is_err(), + "non-integer entry" + ); +} + +// FALSIFY-EG2L-003: layer attention kind from attn_q widths; swapping the shapes swaps the kinds. +#[test] +fn falsify_eg2l_003_layer_kind_from_shapes() { + let widths: Vec = (0..24) + .map(|i| { + if [5, 11, 17, 23].contains(&i) { + 2048 + } else { + 1024 + } + }) + .collect(); + let kinds = layer_attention_kinds(&widths).expect("two widths"); + let globals: Vec = kinds + .iter() + .enumerate() + .filter(|(_, k)| **k == LayerAttention::Global) + .map(|(i, _)| i) + .collect(); + assert_eq!(globals, vec![5, 11, 17, 23]); +} + +#[test] +fn falsify_eg2l_003_swapped_shapes_swap_kinds() { + let widths = [2048, 1024, 1024, 2048, 1024]; + let kinds = layer_attention_kinds(&widths).expect("two widths"); + use LayerAttention::{Global, Local}; + assert_eq!(kinds, vec![Global, Local, Local, Global, Local]); +} + +#[test] +fn falsify_eg2l_003_layer_kind_refusals() { + assert!(layer_attention_kinds(&[]).is_err(), "no layers"); + let err = layer_attention_kinds(&[1024, 1024]).expect_err("single width"); + assert!(err.contains("no local/global split"), "{err}"); + assert!(layer_attention_kinds(&[0, 1024]).is_err(), "zero width"); +} From d53fc9fac027f06f0814afc6dbea6e3d447cb35c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 04:58:58 +0200 Subject: [PATCH 06/21] =?UTF-8?q?feat(embed):=20EG-1=20slice=202=20?= =?UTF-8?q?=E2=80=94=20config=20reads=20head=5Fcount=5Fkv=20per=20layer,?= =?UTF-8?q?=20never=20collapses=20an=20array?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GGUFConfig::from_gguf read attention.head_count_kv through a UInt32-only reader, so an array value fell through to num_heads silently (I-9). It now goes through model_kind::num_kv_heads_per_layer and model_kind::uniform_kv_heads: - absent key: num_heads, as before; scalar: unchanged - uniform array: its value (was num_heads) - non-uniform, wrong-length, zero or non-integer array: refused (InvalidShape) by name Behaviour change outside the embedding path: a GGUF with per-layer head counts (e.g. OpenELM, hybrid files with 0 KV heads on SSM layers) used to load with a guessed num_heads and kv = num_heads; it is now refused instead of run with fabricated head counts. Contract embeddinggemma2-load-v1: EG2L-INV-006 + FALSIFY-EG2L-006. RED observed before the change (uniform array gave 4, non-uniform loaded); two mutants killed (uniform check removed, array ignored). gguf lib suite 2375 passed; clippy -D warnings clean; pv validate 0 errors; ont2b/4b/5/8 + pvl discharge green. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/embeddinggemma2-load-v1.yaml | 13 +++- crates/aprender-serve/src/gguf/config.rs | 11 ++- crates/aprender-serve/src/model_kind.rs | 39 +++++++++- crates/aprender-serve/src/model_kind_tests.rs | 74 +++++++++++++++++++ 4 files changed, 133 insertions(+), 4 deletions(-) diff --git a/contracts/embeddinggemma2-load-v1.yaml b/contracts/embeddinggemma2-load-v1.yaml index e7518c51a4..0168d1db45 100644 --- a/contracts/embeddinggemma2-load-v1.yaml +++ b/contracts/embeddinggemma2-load-v1.yaml @@ -32,7 +32,7 @@ equations: postconditions: - 'no generative forward pass runs on an embedding file' per_layer_kv_heads: - formula: kv(l) = head_count_kv[l] if head_count_kv is an array, else head_count_kv for every l + formula: kv(l) = head_count_kv[l] if head_count_kv is an array, else head_count_kv for every l; config_kv(f) = kv(0) when every kv(l) equals kv(0) domain: l in [0, block_count), head_count_kv read from the GGUF codomain: positive integers invariants: @@ -42,6 +42,7 @@ equations: - block_count > 0 postconditions: - 'len(kv) == block_count and every entry is > 0' + - 'a generative config takes kv only when every layer agrees; otherwise the file is refused' layer_attention_kind: formula: attn(l) = global if width(attn_q[l]) equals the widest attn_q in the file, else local domain: l in [0, block_count), attn_q output widths read from tensor shapes @@ -79,6 +80,11 @@ proof_obligations: property: Generative path refuses by kind formal: "∀ f: kind(f) = embedding ⇒ load_generative(f) = refused(model_kind)" applies_to: all +- id: EG2L-INV-006 + type: invariant + property: Generative config reads KV heads per layer + formal: "∀ f: (∀ l: kv(l) = kv(0)) ⇒ config_kv(f) = kv(0) ∧ ((∃ l: kv(l) ≠ kv(0)) ⇒ refused(f))" + applies_to: all falsification_tests: - id: FALSIFY-EG2L-001 rule: Exact architecture match @@ -105,6 +111,11 @@ falsification_tests: prediction: gemma-embedding2 with attention.causal = true is refused naming the flag; false or absent is embedding test: model_kind::tests::falsify_eg2l_001_kind_and_causal_flag if_fails: A causal embedding file was admitted under either kind +- id: FALSIFY-EG2L-006 + rule: Generative config reads KV heads per layer + prediction: A uniform head_count_kv array gives its value (not num_heads); a non-uniform or wrong-length array is refused naming per-layer; a scalar and an absent key behave as before + test: model_kind::tests::falsify_eg2l_006_* + if_fails: The array collapsed to num_heads or to its first entry in GGUFConfig::from_gguf qa_gate: id: F-EG2L-001 name: embeddinggemma2-load-v1 Contract diff --git a/crates/aprender-serve/src/gguf/config.rs b/crates/aprender-serve/src/gguf/config.rs index 9d981fb6d3..4398d1e43a 100644 --- a/crates/aprender-serve/src/gguf/config.rs +++ b/crates/aprender-serve/src/gguf/config.rs @@ -750,8 +750,15 @@ impl GGUFConfig { // The contract default is architecture-specific (e.g., 1e-5 for LLaMA, 1e-6 for Qwen2). let eps = model.rms_epsilon().unwrap_or(constraints.default_eps); - // num_kv_heads (for GQA - e.g., Qwen uses fewer KV heads than Q heads) - let num_kv_heads = model.num_kv_heads().unwrap_or(num_heads); + // num_kv_heads (for GQA - e.g., Qwen uses fewer KV heads than Q heads). Read per layer, scalar or + // array (APR-EMBED-001 I-9): an array is never collapsed to num_heads, and one whose counts differ + // across layers is refused because this config holds a single count. + let num_kv_heads = match crate::model_kind::num_kv_heads_per_layer(model) { + None => num_heads, + Some(per_layer) => per_layer + .and_then(|v| crate::model_kind::uniform_kv_heads(&v)) + .map_err(|reason| RealizarError::InvalidShape { reason })?, + }; // GH-305: Infer head_dim from GGUF metadata or tensor shapes. let explicit_head_dim = Self::infer_explicit_head_dim(model, hidden_dim, num_heads); diff --git a/crates/aprender-serve/src/model_kind.rs b/crates/aprender-serve/src/model_kind.rs index 43ccb59775..7716c87f75 100644 --- a/crates/aprender-serve/src/model_kind.rs +++ b/crates/aprender-serve/src/model_kind.rs @@ -10,7 +10,7 @@ //! //! Contract: `contracts/embeddinggemma2-load-v1.yaml`. -use crate::gguf::GGUFValue; +use crate::gguf::{GGUFModel, GGUFValue}; /// The GGUF `general.architecture` string of EmbeddingGemma 2, as the files carry it. pub const EMBEDDING_GEMMA2_ARCH: &str = "gemma-embedding2"; @@ -93,6 +93,43 @@ fn kv_count(value: &GGUFValue) -> Result { } } +/// Per-layer KV-head counts of a GGUF file, from `attention.head_count_kv`, scalar or array (I-9). +/// +/// `None` when the key is absent. `Some(Err)` when the value cannot be read per layer: an array of the +/// wrong length, a zero or non-integer count, or a missing `block_count`. +#[must_use] +pub fn num_kv_heads_per_layer(model: &GGUFModel) -> Option, String>> { + let arch = model.architecture()?; + let key = crate::gguf::keys::arch_key(arch, crate::gguf::keys::ATTENTION_HEAD_COUNT_KV); + let value = model.metadata.get(&key)?; + let Some(block_count) = model.num_layers() else { + return Some(Err( + "block_count missing: cannot read per-layer KV heads".to_string() + )); + }; + Some(per_layer_kv_heads(value, block_count)) +} + +/// The single KV-head count of a model whose layers all share one, from the per-layer counts. +/// +/// A generative config carries one KV-head count. Per-layer counts that differ cannot be represented there +/// and are refused by name, never collapsed to the first entry or to the query head count (I-9). +/// +/// # Errors +/// Returns the reason when the list is empty or its counts differ. +pub fn uniform_kv_heads(per_layer: &[usize]) -> Result { + let Some(&first) = per_layer.first() else { + return Err("no layers: cannot read the KV-head count".to_string()); + }; + if per_layer.iter().any(|&k| k != first) { + return Err(format!( + "per-layer attention.head_count_kv differs across layers ({per_layer:?}); a config with one \ + KV-head count cannot represent it (I-9)" + )); + } + Ok(first) +} + /// The attention kind of one layer. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LayerAttention { diff --git a/crates/aprender-serve/src/model_kind_tests.rs b/crates/aprender-serve/src/model_kind_tests.rs index 13a6202937..d9faa782f4 100644 --- a/crates/aprender-serve/src/model_kind_tests.rs +++ b/crates/aprender-serve/src/model_kind_tests.rs @@ -141,3 +141,77 @@ fn falsify_eg2l_003_layer_kind_refusals() { assert!(err.contains("no local/global split"), "{err}"); assert!(layer_attention_kinds(&[0, 1024]).is_err(), "zero width"); } + +// FALSIFY-EG2L-006: the config loader reads head_count_kv through the per-layer reader. A uniform +// array gives its value; a non-uniform array is refused by name, never collapsed to num_heads. +fn kv_fixture(kv: Option<&[i32]>, scalar: Option, layers: u32) -> Vec { + use crate::gguf::test_factory::GGUFBuilder; + let mut b = GGUFBuilder::new() + .architecture("llama") + .hidden_dim("llama", 64) + .num_layers("llama", layers) + .num_heads("llama", 4) + .add_f32_tensor("token_embd.weight", &[8, 64], &[0.0f32; 512]); + if let Some(arr) = kv { + b = b.add_i32_array("llama.attention.head_count_kv", arr); + } + if let Some(k) = scalar { + b = b.num_kv_heads("llama", k); + } + b.build() +} + +fn config_of(data: &[u8]) -> crate::error::Result { + let model = crate::gguf::GGUFModel::from_bytes(data).expect("fixture parses"); + crate::gguf::GGUFConfig::from_gguf(&model) +} + +#[test] +fn falsify_eg2l_006_scalar_kv_heads_unchanged() { + let cfg = config_of(&kv_fixture(None, Some(2), 3)).expect("scalar loads"); + assert_eq!(cfg.num_kv_heads, 2); + let cfg = config_of(&kv_fixture(None, None, 3)).expect("absent loads"); + assert_eq!( + cfg.num_kv_heads, 4, + "absent key keeps the num_heads default" + ); +} + +#[test] +fn falsify_eg2l_006_uniform_kv_array_gives_its_value() { + let cfg = config_of(&kv_fixture(Some(&[2, 2, 2]), None, 3)).expect("uniform array loads"); + assert_eq!( + cfg.num_kv_heads, 2, + "a uniform array must not collapse to num_heads (4)" + ); +} + +#[test] +fn falsify_eg2l_006_non_uniform_kv_array_refused() { + let err = + config_of(&kv_fixture(Some(&[2, 1, 2]), None, 3)).expect_err("non-uniform must be refused"); + let msg = err.to_string(); + assert!(msg.contains("per-layer"), "{msg}"); + let err = + config_of(&kv_fixture(Some(&[2, 2]), None, 3)).expect_err("wrong length must be refused"); + assert!(err.to_string().contains("2 entries for 3 blocks"), "{err}"); +} + +#[test] +fn falsify_eg2l_006_metadata_exposes_per_layer_values() { + let data = kv_fixture(Some(&[2, 1, 2]), None, 3); + let model = crate::gguf::GGUFModel::from_bytes(&data).expect("fixture parses"); + assert_eq!(num_kv_heads_per_layer(&model), Some(Ok(vec![2, 1, 2]))); + let data = kv_fixture(None, None, 3); + let model = crate::gguf::GGUFModel::from_bytes(&data).expect("fixture parses"); + assert_eq!(num_kv_heads_per_layer(&model), None); +} + +#[test] +fn uniform_kv_heads_cases() { + assert_eq!(uniform_kv_heads(&[3, 3]), Ok(3)); + assert!(uniform_kv_heads(&[3, 1]) + .expect_err("mixed") + .contains("per-layer")); + assert!(uniform_kv_heads(&[]).is_err()); +} From 2dd924d486f21f61fd72f78225c7c8ffd2a35ea2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 05:35:14 +0200 Subject: [PATCH 07/21] =?UTF-8?q?feat(embed):=20EG-1=20slice=203=20?= =?UTF-8?q?=E2=80=94=20chat=20and=20serve=20refuse=20an=20embedding=20mode?= =?UTF-8?q?l=20by=20kind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `apr chat` on a gemma-embedding2 GGUF exited 0. The contract gate's model_kind refusal ran only inside the CUDA preload, and a failure there printed `will use CPU` and continued; the CPU path ran no gate, so an empty stdin ended the session cleanly. One refusal, `commands::model_kind_gate::refuse_non_generative`, now runs in `ChatSession::new` before any tokenizer or backend is built, and at the top of `build_serve_model`, which every GGUF serve route goes through. Both exit 6 (ModelLoadFailed) and name `apr embed` (I-3). A causal embedding file is refused by name. Generative files and headers that do not parse go on to the loaders unchanged. realizar gains `model_kind::gguf_model_kind`, which reads the kind from a parsed file. FALSIFY-EG2L-007 / EG2L-INV-007 in embeddinggemma2-load-v1. RED before wiring: chat exited 4 on the missing tokenizer, and serve was refused over missing metadata; neither named the kind. GREEN 5/5. apr-cli chat+serve 723 passed; model_kind 14/14; clippy -D warnings clean; fmt clean; pv validate 0 errors; ont2b/4b/5/8 and pvl discharge green. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/embeddinggemma2-load-v1.yaml | 10 ++++ .../src/commands/chat_model_kind_eg1_tests.rs | 18 ++++++ crates/apr-cli/src/commands/chat_session.rs | 6 ++ .../apr-cli/src/commands/chat_session_02.rs | 6 ++ crates/apr-cli/src/commands/mod.rs | 1 + .../apr-cli/src/commands/model_kind_gate.rs | 36 ++++++++++++ .../src/commands/model_kind_gate_tests.rs | 57 +++++++++++++++++++ .../commands/serve/handler_gpu_completion.rs | 24 ++++++++ crates/aprender-serve/src/model_kind.rs | 17 ++++++ 9 files changed, 175 insertions(+) create mode 100644 crates/apr-cli/src/commands/chat_model_kind_eg1_tests.rs create mode 100644 crates/apr-cli/src/commands/model_kind_gate.rs create mode 100644 crates/apr-cli/src/commands/model_kind_gate_tests.rs diff --git a/contracts/embeddinggemma2-load-v1.yaml b/contracts/embeddinggemma2-load-v1.yaml index 0168d1db45..e46ac68c6b 100644 --- a/contracts/embeddinggemma2-load-v1.yaml +++ b/contracts/embeddinggemma2-load-v1.yaml @@ -85,6 +85,11 @@ proof_obligations: property: Generative config reads KV heads per layer formal: "∀ f: (∀ l: kv(l) = kv(0)) ⇒ config_kv(f) = kv(0) ∧ ((∃ l: kv(l) ≠ kv(0)) ⇒ refused(f))" applies_to: all +- id: EG2L-INV-007 + type: invariant + property: Generative verbs refuse by kind on every host + formal: "∀ f, v ∈ {chat, serve}: kind(f) = embedding ⇒ exit(apr v f) = 6 ∧ names(apr embed)" + applies_to: all falsification_tests: - id: FALSIFY-EG2L-001 rule: Exact architecture match @@ -116,6 +121,11 @@ falsification_tests: prediction: A uniform head_count_kv array gives its value (not num_heads); a non-uniform or wrong-length array is refused naming per-layer; a scalar and an absent key behave as before test: model_kind::tests::falsify_eg2l_006_* if_fails: The array collapsed to num_heads or to its first entry in GGUFConfig::from_gguf +- id: FALSIFY-EG2L-007 + rule: Generative verbs refuse by kind on every host + prediction: apr chat and apr serve on a gemma-embedding2 GGUF fail with ModelLoadFailed (exit 6) naming apr embed, before a tokenizer or backend is built and with no GPU; generative and unparsed files pass through to the loaders + test: falsify_eg2l_007_* + if_fails: The refusal ran only inside the CUDA preload, so the CPU chat path ended an empty session with exit 0 qa_gate: id: F-EG2L-001 name: embeddinggemma2-load-v1 Contract diff --git a/crates/apr-cli/src/commands/chat_model_kind_eg1_tests.rs b/crates/apr-cli/src/commands/chat_model_kind_eg1_tests.rs new file mode 100644 index 0000000000..97f0660793 --- /dev/null +++ b/crates/apr-cli/src/commands/chat_model_kind_eg1_tests.rs @@ -0,0 +1,18 @@ +// FALSIFY-EG2L-007 at the call site: `ChatSession::new` refuses an embedding GGUF on every host, +// before a tokenizer or backend is built. Delete the call in `ChatSession::new` and this goes RED. + +#[test] +fn falsify_eg2l_007_chat_session_new_refuses_embedding_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("embed.gguf"); + std::fs::write( + &path, + crate::commands::model_kind_gate::tests::eg1_gguf("gemma-embedding2", Some(false)), + ) + .expect("write"); + let Err(err) = ChatSession::new(&path, true) else { + panic!("ChatSession::new must refuse an embedding file"); + }; + assert_eq!(err.exit_code_value(), 6, "{err}"); + assert!(err.to_string().contains("apr embed"), "{err}"); +} diff --git a/crates/apr-cli/src/commands/chat_session.rs b/crates/apr-cli/src/commands/chat_session.rs index 55db036e33..bccbf7b61a 100644 --- a/crates/apr-cli/src/commands/chat_session.rs +++ b/crates/apr-cli/src/commands/chat_session.rs @@ -78,6 +78,12 @@ include!("chat_generate_safetensors.rs"); use super::*; include!("chat_engine_identity_4263.rs"); } + + #[cfg(test)] + mod model_kind_eg1 { + use super::*; + include!("chat_model_kind_eg1_tests.rs"); + } } diff --git a/crates/apr-cli/src/commands/chat_session_02.rs b/crates/apr-cli/src/commands/chat_session_02.rs index 4b82720af4..2a78befe8b 100644 --- a/crates/apr-cli/src/commands/chat_session_02.rs +++ b/crates/apr-cli/src/commands/chat_session_02.rs @@ -174,6 +174,12 @@ impl ChatSession { report_loaded_format(format, &model_bytes, start.elapsed()); + // APR-EMBED-001 EG-1 (I-3): an embedding model is refused by kind before any tokenizer + // or backend is built, on every host; the CUDA preload was the only place it was refused. + if format == ModelFormat::Gguf { + crate::commands::model_kind_gate::refuse_non_generative_bytes(&model_bytes, "chat")?; + } + let (llama_tokenizer, qwen_tokenizer) = load_tokenizers(format, &model_bytes, path)?; if matches!( diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index ca651f62ad..357cdc96f6 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -107,6 +107,7 @@ pub(crate) mod mcp; pub(crate) mod merge; #[cfg(feature = "training")] pub(crate) mod model_config; +pub(crate) mod model_kind_gate; // #3661: a model parse failure names the format its magic bytes identify. #[cfg(test)] mod model_file_error_tests_3661; diff --git a/crates/apr-cli/src/commands/model_kind_gate.rs b/crates/apr-cli/src/commands/model_kind_gate.rs new file mode 100644 index 0000000000..eb33c15672 --- /dev/null +++ b/crates/apr-cli/src/commands/model_kind_gate.rs @@ -0,0 +1,36 @@ +//! APR-EMBED-001 EG-1 (I-3): the generative verbs refuse an embedding model by kind and name the verb that +//! serves it. Contract: `contracts/embeddinggemma2-load-v1.yaml` (FALSIFY-EG2L-007). + +use crate::error::CliError; +use realizar::gguf::GGUFModel; +use realizar::model_kind::{gguf_model_kind, ModelKind}; + +/// Refuse a GGUF file that is not a generative model, before a tokenizer or backend is built. +/// +/// # Errors +/// `CliError::ModelLoadFailed` (exit 6) for an embedding model, naming `apr embed`, and for a file whose +/// causal flag contradicts its architecture. +pub(crate) fn refuse_non_generative(model: &GGUFModel, verb: &str) -> Result<(), CliError> { + match gguf_model_kind(model) { + Ok(ModelKind::Generative) => Ok(()), + Ok(ModelKind::Embedding) => Err(CliError::ModelLoadFailed(format!( + "'{}' is an embedding model (ModelKind::Embedding): it has no language-model head, so \ + `apr {verb}` cannot serve it. Use `apr embed`.", + model.architecture().unwrap_or_default() + ))), + Err(reason) => Err(CliError::ModelLoadFailed(reason)), + } +} + +/// [`refuse_non_generative`] on raw file bytes. A header that does not parse is left to the loaders, +/// which keep their own errors. +pub(crate) fn refuse_non_generative_bytes(model_bytes: &[u8], verb: &str) -> Result<(), CliError> { + match GGUFModel::from_bytes(model_bytes) { + Ok(model) => refuse_non_generative(&model, verb), + Err(_) => Ok(()), + } +} + +#[cfg(test)] +#[path = "model_kind_gate_tests.rs"] +pub(crate) mod tests; diff --git a/crates/apr-cli/src/commands/model_kind_gate_tests.rs b/crates/apr-cli/src/commands/model_kind_gate_tests.rs new file mode 100644 index 0000000000..0a5f2acb47 --- /dev/null +++ b/crates/apr-cli/src/commands/model_kind_gate_tests.rs @@ -0,0 +1,57 @@ +// FALSIFY-EG2L-007: a generative verb on an embedding GGUF exits 6 (ModelLoadFailed) naming +// `apr embed`. Before EG-1, `apr chat` ran no gate on the CPU path and an empty stdin exited 0. + +use super::*; + +pub(crate) fn eg1_gguf(arch: &str, causal: Option) -> Vec { + use aprender::format::gguf::{export_tensors_to_gguf, GgmlType, GgufTensor, GgufValue}; + let tensors = vec![GgufTensor { + name: "token_embd.weight".to_string(), + shape: vec![4, 8], + dtype: GgmlType::F32, + data: vec![0u8; 4 * 8 * 4], + }]; + let mut metadata = vec![( + "general.architecture".to_string(), + GgufValue::String(arch.to_string()), + )]; + if let Some(c) = causal { + metadata.push((format!("{arch}.attention.causal"), GgufValue::Bool(c))); + } + let mut bytes = Vec::new(); + export_tensors_to_gguf(&mut bytes, &tensors, &metadata).expect("write GGUF"); + bytes +} + +#[test] +fn falsify_eg2l_007_embedding_refused_by_kind() { + for causal in [None, Some(false)] { + let err = refuse_non_generative_bytes(&eg1_gguf("gemma-embedding2", causal), "chat") + .expect_err("an embedding file must not reach a generative verb"); + assert_eq!(err.exit_code_value(), 6, "{err}"); + let msg = err.to_string(); + assert!(msg.contains("Use `apr embed`"), "{msg}"); + assert!(msg.contains("`apr chat`"), "{msg}"); + assert!(!msg.contains("Gemma3"), "{msg}"); + } +} + +#[test] +fn falsify_eg2l_007_causal_contradiction_refused() { + let err = refuse_non_generative_bytes(&eg1_gguf("gemma-embedding2", Some(true)), "serve") + .expect_err("a causal embedding file is refused"); + assert_eq!(err.exit_code_value(), 6, "{err}"); + assert!(err.to_string().contains("attention.causal = true"), "{err}"); +} + +#[test] +fn falsify_eg2l_007_generative_and_unparsed_files_pass_through() { + for arch in ["llama", "gemma2", "gemma-embedding3"] { + assert!( + refuse_non_generative_bytes(&eg1_gguf(arch, None), "chat").is_ok(), + "{arch}" + ); + } + let bytes = eg1_gguf("gemma-embedding2", None); + assert!(refuse_non_generative_bytes(&bytes[..20], "chat").is_ok()); +} diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index a5d45927f4..a3c4af2573 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -463,6 +463,9 @@ fn build_serve_model( mapped_model: &realizar::gguf::MappedGGUFModel, ) -> Result { use realizar::gguf::OwnedQuantizedModel; + // APR-EMBED-001 EG-1 (I-3): every GGUF route comes through here, so an embedding model is refused + // by kind once, naming the verb that serves it. + crate::commands::model_kind_gate::refuse_non_generative(&mapped_model.model, "serve")?; // #3571: the Qwen3.5 hybrid (Gated Delta Net) has no dense layers, so `from_mapped` refuses // it by name and `apr serve` could not load the architecture the last release shipped — while // `apr run` loaded it fine, because `run_gguf_inference` has carried exactly this branch since @@ -918,6 +921,27 @@ mod zero_layer_refusal_tests { } } + /// FALSIFY-EG2L-007 (APR-EMBED-001 EG-1, I-3): every GGUF serve route refuses an embedding + /// model by kind, naming `apr embed`. Delete the call in `build_serve_model` and this goes RED. + #[test] + fn falsify_eg2l_007_serve_refuses_embedding_by_kind() { + let file = tempfile::NamedTempFile::with_suffix(".gguf").expect("temp file"); + std::fs::write( + file.path(), + crate::commands::model_kind_gate::tests::eg1_gguf("gemma-embedding2", Some(false)), + ) + .expect("write"); + let mapped = realizar::gguf::MappedGGUFModel::from_path(file.path()).expect("map"); + match build_serve_model(&mapped) { + Err(CliError::ModelLoadFailed(msg)) => { + assert!(msg.contains("`apr serve`"), "{msg}"); + assert!(msg.contains("Use `apr embed`"), "{msg}"); + } + Err(other) => panic!("refused for the wrong reason: {other}"), + Ok(model) => panic!("an embedding file reached the serve routes ({} layers)", model.layers().len()), + } + } + /// The load path itself refuses, before any route is chosen. Delete the check in /// `build_serve_model` and this goes RED: the zero-layer base loads. Needs the real /// Qwen3.5 file, whose base IS a zero-layer stack. diff --git a/crates/aprender-serve/src/model_kind.rs b/crates/aprender-serve/src/model_kind.rs index 7716c87f75..5404e2dfe0 100644 --- a/crates/aprender-serve/src/model_kind.rs +++ b/crates/aprender-serve/src/model_kind.rs @@ -50,6 +50,23 @@ pub fn model_kind(arch: &str, causal: Option) -> Result } } +/// The kind of a parsed GGUF file, from `general.architecture` and `.attention.causal`. +/// +/// A file with no architecture string is generative: the generative loaders keep their own refusals for it. +/// +/// # Errors +/// Returns the reason when the architecture and the causal flag disagree (see [`model_kind`]). +pub fn gguf_model_kind(model: &GGUFModel) -> Result { + let Some(arch) = model.architecture() else { + return Ok(ModelKind::Generative); + }; + let causal = match model.metadata.get(&format!("{arch}.attention.causal")) { + Some(GGUFValue::Bool(b)) => Some(*b), + _ => None, + }; + model_kind(arch, causal) +} + /// The KV-head count of every layer, from the `attention.head_count_kv` value. /// /// A scalar applies to every layer. An array gives one count per layer and must be exactly `block_count` long. From 391318a78417bd61177cf9a5f808b2f32ab691d6 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 06:46:33 +0200 Subject: [PATCH 08/21] =?UTF-8?q?feat(embed):=20EG-1=20slice=204=20?= =?UTF-8?q?=E2=80=94=20SPM-style=20BPE=20tokenizer=20for=20gemma4=20vocabu?= =?UTF-8?q?laries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gemma4 GGUF (tokenizer.ggml.model = gemma4, 262,144 tokens plus merges) fell to greedy longest-match, so its ids departed from llama.cpp's. This ports llama.cpp's gemma4 BPE session: specials first, space -> U+2581 with no prefix, [^\n]+|[\n]+ runs, merges keyed by text (lowest rank, leftmost on ties, stale pairs skipped), unknown symbols to <0xXX> byte tokens. - gguf/spm_bpe.rs + 12-row case table (FALSIFY-EG2L-008) - GGUFModel::encode tries SPM-style BPE, then byte-level BPE, then greedy - apr tokenize reports canonical for gemma4; tokenizer_parity.sh matches the new uncovered message - metadata.rs get_tensor_f32: the per-qtype arms become one block table + tensor_byte_range, behaviour unchanged. token.rs is include!d next to it, so its cognitive complexity (66) blocked any token.rs commit at the hook cargo test -p aprender-serve --lib -- gguf tensor encode token: 4124 passed, 0 failed. clippy -D warnings clean on aprender-serve and apr-cli. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- contracts/embeddinggemma2-load-v1.yaml | 22 + .../apr-cli/src/commands/tokenize_encode.rs | 10 +- crates/aprender-serve/src/gguf/metadata.rs | 426 +++--------------- crates/aprender-serve/src/gguf/mod.rs | 2 + crates/aprender-serve/src/gguf/spm_bpe.rs | 338 ++++++++++++++ .../aprender-serve/src/gguf/spm_bpe_tests.rs | 218 +++++++++ crates/aprender-serve/src/gguf/token.rs | 142 +++--- scripts/tokenizer_parity.sh | 2 +- scripts/tokenizer_parity_lib.sh | 2 +- 9 files changed, 736 insertions(+), 426 deletions(-) create mode 100644 crates/aprender-serve/src/gguf/spm_bpe.rs create mode 100644 crates/aprender-serve/src/gguf/spm_bpe_tests.rs diff --git a/contracts/embeddinggemma2-load-v1.yaml b/contracts/embeddinggemma2-load-v1.yaml index e46ac68c6b..bcd2ba7043 100644 --- a/contracts/embeddinggemma2-load-v1.yaml +++ b/contracts/embeddinggemma2-load-v1.yaml @@ -54,6 +54,18 @@ equations: - every block has an attn_q tensor of non-zero width postconditions: - 'at least one layer is global and at least one is local' + spm_bpe_encode: + formula: ids(t) = concat over fragments of special id, or BPE(w) for each run w of [^\n]+|[\n]+ in escape(t), where BPE merges pairs by text in ascending merge rank, leftmost on ties + domain: t a UTF-8 text; a GGUF with tokenizer.ggml.model = gemma4 and tokenizer.ggml.merges + codomain: sequences of token ids in [0, n_vocab) + invariants: + - escape replaces every space with U+2581 and prepends nothing + - a newline run that is itself a token is one symbol; otherwise every UTF-8 character starts as one symbol + - a final symbol that is not a token is emitted as its <0xXX> byte tokens, and a byte with no token is dropped + preconditions: + - tokenizer.ggml.model is gemma4 and tokenizer.ggml.merges is present + postconditions: + - ids equal the pinned llama.cpp tokenizer ids for the same text and file proof_obligations: - id: EG2L-INV-001 type: invariant @@ -90,6 +102,11 @@ proof_obligations: property: Generative verbs refuse by kind on every host formal: "∀ f, v ∈ {chat, serve}: kind(f) = embedding ⇒ exit(apr v f) = 6 ∧ names(apr embed)" applies_to: all +- id: EG2L-INV-008 + type: invariant + property: SPM-style BPE ids equal the reference tokenizer + formal: "∀ t, f: model(f) = gemma4 ⇒ encode_f(t) = llama_tokenize_f(t)" + applies_to: all falsification_tests: - id: FALSIFY-EG2L-001 rule: Exact architecture match @@ -126,6 +143,11 @@ falsification_tests: prediction: apr chat and apr serve on a gemma-embedding2 GGUF fail with ModelLoadFailed (exit 6) naming apr embed, before a tokenizer or backend is built and with no GPU; generative and unparsed files pass through to the loaders test: falsify_eg2l_007_* if_fails: The refusal ran only inside the CUDA preload, so the CPU chat path ended an empty session with exit 0 +- id: FALSIFY-EG2L-008 + rule: SPM-style BPE ids equal the reference tokenizer + prediction: Merges apply by text, lowest rank first and leftmost on ties; stale pairs are skipped; a space becomes U+2581 with no prefix; words split at newline runs; unknown symbols become <0xXX> tokens; specials split out first; a gemma4 file without merges is refused by name + test: gguf::spm_bpe::tests::* + if_fails: The gemma4 vocabulary fell back to greedy longest-match, or a merge order, space or byte rule departs from llama.cpp; scripts/tokenizer_parity.sh reports the first id mismatch on a real file qa_gate: id: F-EG2L-001 name: embeddinggemma2-load-v1 Contract diff --git a/crates/apr-cli/src/commands/tokenize_encode.rs b/crates/apr-cli/src/commands/tokenize_encode.rs index 79391c81ad..49a86ddbf3 100644 --- a/crates/apr-cli/src/commands/tokenize_encode.rs +++ b/crates/apr-cli/src/commands/tokenize_encode.rs @@ -3,7 +3,8 @@ //! The ids come from `GGUFModel::encode`, the function `apr run`, `apr serve` and //! `apr parity` tokenize prompts with, so a regression in how that function routes a //! vocabulary shows up here too. The output names the path that produced the ids: the -//! canonical byte-level BPE (pre-tokenizer plus ranked merges, identical to llama.cpp), or the +//! canonical byte-level BPE (pre-tokenizer plus ranked merges, identical to llama.cpp) or SPM-style +//! BPE (`gemma4`, APR-EMBED-001 EG-1), or the //! greedy longest-match fallback and why it was taken. `scripts/tokenizer_parity.sh` compares //! these ids with the pinned llama.cpp's and refuses a fallback. //! @@ -62,9 +63,14 @@ pub(crate) fn run_encode( Ok(_) => "canonical".to_string(), Err(refusal) => format!("greedy-fallback: {refusal}"), } + } else if realizar::gguf::spm_bpe::is_spm_bpe(&mapped.model.metadata) { + match realizar::gguf::spm_bpe::SpmBpe::from_gguf(&mapped.model.metadata, &vocab) { + Ok(_) => "canonical".to_string(), + Err(refusal) => format!("greedy-fallback: {refusal}"), + } } else { format!( - "greedy-fallback: tokenizer.ggml.model is {:?}, not a byte-level vocabulary", + "greedy-fallback: tokenizer.ggml.model is {:?}, not a byte-level or SPM-style BPE vocabulary", tokenizer_model.as_deref().unwrap_or("absent") ) }; diff --git a/crates/aprender-serve/src/gguf/metadata.rs b/crates/aprender-serve/src/gguf/metadata.rs index 0aaa9c060f..f4a0ec5129 100644 --- a/crates/aprender-serve/src/gguf/metadata.rs +++ b/crates/aprender-serve/src/gguf/metadata.rs @@ -55,372 +55,51 @@ impl GGUFModel { })?; let offset = self.tensor_data_start + tensor_offset; - // Extract and dequantize based on qtype + // Extract and dequantize based on qtype. A block type reads whole blocks and trims the + // padding the last block carries; every type checks its byte range against the file first. + use crate::quantize::{ + dequantize_f16, dequantize_q2_k, dequantize_q3_k, dequantize_q4_0, dequantize_q4_1, + dequantize_q4_k_simd, dequantize_q5_0, dequantize_q5_1, dequantize_q5_k, + dequantize_q6_k, dequantize_q8_0_simd, QK_K, + }; + type Dequant = fn(&[u8]) -> Result>; + // (elements per block, bytes per block, dequantizer) + let blocked: Option<(usize, usize, Dequant)> = match tensor.qtype { + GGUF_TYPE_Q4_0 => Some((32, 18, dequantize_q4_0)), + GGUF_TYPE_Q8_0 => Some((32, 34, dequantize_q8_0_simd)), + GGUF_TYPE_Q2_K => Some((QK_K, 84, dequantize_q2_k)), + GGUF_TYPE_Q3_K => Some((QK_K, 110, dequantize_q3_k)), + GGUF_TYPE_Q4_K => Some((QK_K, 144, dequantize_q4_k_simd)), + GGUF_TYPE_Q5_K => Some((QK_K, 176, dequantize_q5_k)), + GGUF_TYPE_Q6_K => Some((QK_K, 210, dequantize_q6_k)), + GGUF_TYPE_Q4_1 => Some((32, 20, dequantize_q4_1)), + GGUF_TYPE_Q5_0 => Some((32, 22, dequantize_q5_0)), + GGUF_TYPE_Q5_1 => Some((32, 24, dequantize_q5_1)), + _ => None, + }; + if let Some((block_elems, block_bytes, dequant)) = blocked { + let bytes = tensor_byte_range(file_data, offset, size.div_ceil(block_elems) * block_bytes)?; + let mut values = dequant(bytes)?; + values.truncate(size); + return Ok(values); + } match tensor.qtype { - GGUF_TYPE_F32 => { - // Unquantized F32 data - let byte_size = size * 4; // 4 bytes per f32 - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let values = bytes - .as_chunks::<4>() - .0 - .iter() - .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect(); - Ok(values) - }, - GGUF_TYPE_Q4_0 => { - // Q4_0 quantized data - use crate::quantize::dequantize_q4_0; - - // Q4_0 block: 32 elements - // Layout: 1×f16 scale (2 bytes) + 16 bytes (32×4-bit values) = 18 bytes - const BLOCK_BYTES: usize = 18; - const BLOCK_SIZE: usize = 32; - - let num_blocks = size.div_ceil(BLOCK_SIZE); - let byte_size = num_blocks * BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q4_0(bytes)?; - - // Trim to exact size (dequantization pads to block boundaries) - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q8_0 => { - // Q8_0 quantized data - use SIMD-parallel for faster loading - use crate::quantize::dequantize_q8_0_simd; - - // Q8_0 block size: 34 bytes (2 for f16 scale + 32 for quants) - const BLOCK_BYTES: usize = 34; - const BLOCK_SIZE: usize = 32; - - let num_blocks = size.div_ceil(BLOCK_SIZE); - let byte_size = num_blocks * BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q8_0_simd(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q2_K => { - // Q2_K quantized data (K-quantization) - 2 bits per weight - use crate::quantize::{dequantize_q2_k, QK_K}; - - // Q2_K super-block size: 84 bytes for 256 values - const SUPER_BLOCK_BYTES: usize = 84; - - let num_super_blocks = size.div_ceil(QK_K); - let byte_size = num_super_blocks * SUPER_BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q2_k(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q3_K => { - // Q3_K quantized data (K-quantization) - 3 bits per weight - use crate::quantize::{dequantize_q3_k, QK_K}; - - // Q3_K super-block size: 110 bytes for 256 values - const SUPER_BLOCK_BYTES: usize = 110; - - let num_super_blocks = size.div_ceil(QK_K); - let byte_size = num_super_blocks * SUPER_BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q3_k(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q4_K => { - // Q4_K quantized data (K-quantization) - use SIMD-parallel for faster loading - use crate::quantize::{dequantize_q4_k_simd, QK_K}; - - // Q4_K super-block size: 144 bytes for 256 values - const SUPER_BLOCK_BYTES: usize = 144; - - let num_super_blocks = size.div_ceil(QK_K); - let byte_size = num_super_blocks * SUPER_BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q4_k_simd(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q5_K => { - // Q5_K quantized data (K-quantization) - use crate::quantize::{dequantize_q5_k, QK_K}; - - // Q5_K super-block size: 176 bytes for 256 values - const SUPER_BLOCK_BYTES: usize = 176; - - let num_super_blocks = size.div_ceil(QK_K); - let byte_size = num_super_blocks * SUPER_BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q5_k(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q6_K => { - // Q6_K quantized data (K-quantization) - use crate::quantize::{dequantize_q6_k, QK_K}; - - // Q6_K super-block size: 210 bytes for 256 values - const SUPER_BLOCK_BYTES: usize = 210; - - let num_super_blocks = size.div_ceil(QK_K); - let byte_size = num_super_blocks * SUPER_BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q6_k(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_F16 => { - // F16 (half-precision float) data - use crate::quantize::dequantize_f16; - - let byte_size = size * 2; // 2 bytes per f16 - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let values = dequantize_f16(bytes)?; - Ok(values) - }, - GGUF_TYPE_BF16 => { - // BF16 (bfloat16): 2 bytes/elem, no block structure. Reuses the - // existing SIMD converter (y = from_bits((bits as u32) << 16)) - // already used by the safetensors loaders + gguf/embedding.rs. - // #1893-class fix: without this arm a BF16 GGUF's - // embeddings/norms/lm_head hit the catch-all "Unsupported - // quantization type: 30". - let byte_size = size * 2; - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - Ok(crate::inference::simd_bf16_to_f32( - &file_data[offset..offset + byte_size], - )) - }, - GGUF_TYPE_Q4_1 => { - // Q4_1 quantized data - use crate::quantize::dequantize_q4_1; - - // Q4_1 block size: 20 bytes (2 for scale + 2 for min + 16 for quants) - const BLOCK_BYTES: usize = 20; - const BLOCK_SIZE: usize = 32; - - let num_blocks = size.div_ceil(BLOCK_SIZE); - let byte_size = num_blocks * BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q4_1(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q5_0 => { - // Q5_0 quantized data - use crate::quantize::dequantize_q5_0; - - // Q5_0 block size: 22 bytes (2 for scale + 4 for high bits + 16 for quants) - const BLOCK_BYTES: usize = 22; - const BLOCK_SIZE: usize = 32; - - let num_blocks = size.div_ceil(BLOCK_SIZE); - let byte_size = num_blocks * BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q5_0(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, - GGUF_TYPE_Q5_1 => { - // Q5_1 quantized data - use crate::quantize::dequantize_q5_1; - - // Q5_1 block size: 24 bytes (2 for scale + 2 for min + 4 for high bits + 16 for quants) - const BLOCK_BYTES: usize = 24; - const BLOCK_SIZE: usize = 32; - - let num_blocks = size.div_ceil(BLOCK_SIZE); - let byte_size = num_blocks * BLOCK_BYTES; - - if offset + byte_size > file_data.len() { - return Err(RealizarError::UnsupportedOperation { - operation: "get_tensor_f32".to_string(), - reason: format!( - "Data range [{}, {}) exceeds file size {}", - offset, - offset + byte_size, - file_data.len() - ), - }); - } - - let bytes = &file_data[offset..offset + byte_size]; - let mut values = dequantize_q5_1(bytes)?; - - // Trim to exact size - values.truncate(size); - Ok(values) - }, + GGUF_TYPE_F32 => Ok(tensor_byte_range(file_data, offset, size * 4)? + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect()), + GGUF_TYPE_F16 => dequantize_f16(tensor_byte_range(file_data, offset, size * 2)?), + // BF16 (bfloat16): 2 bytes/elem, no block structure. Reuses the existing SIMD + // converter (y = from_bits((bits as u32) << 16)) already used by the safetensors + // loaders + gguf/embedding.rs. #1893-class fix: without this arm a BF16 GGUF's + // embeddings/norms/lm_head hit the catch-all "Unsupported quantization type: 30". + GGUF_TYPE_BF16 => Ok(crate::inference::simd_bf16_to_f32(tensor_byte_range( + file_data, + offset, + size * 2, + )?)), _ => Err(RealizarError::UnsupportedOperation { operation: "get_tensor_f32".to_string(), reason: format!("Unsupported quantization type: {}", tensor.qtype), @@ -612,3 +291,20 @@ impl GGUFModel { } } } + +/// `file_data[offset..offset + byte_size]`, or the `get_tensor_f32` refusal naming the range when +/// it runs past the end of the file. +fn tensor_byte_range(file_data: &[u8], offset: usize, byte_size: usize) -> Result<&[u8]> { + if offset + byte_size > file_data.len() { + return Err(RealizarError::UnsupportedOperation { + operation: "get_tensor_f32".to_string(), + reason: format!( + "Data range [{}, {}) exceeds file size {}", + offset, + offset + byte_size, + file_data.len() + ), + }); + } + Ok(&file_data[offset..offset + byte_size]) +} diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index 64108eb871..dfae41a33d 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -69,6 +69,8 @@ mod wgpu_model; /// #3726: canonical byte-level BPE (pre-tokenizer + ranked merges) for `gpt2` vocabularies. pub mod byte_level_bpe; pub mod ops; +/// APR-EMBED-001 EG-1: canonical SPM-style BPE (ranked merges over raw UTF-8) for `gemma4` vocabularies. +pub mod spm_bpe; // Test helpers module - shared utilities for GGUF tests #[cfg(test)] diff --git a/crates/aprender-serve/src/gguf/spm_bpe.rs b/crates/aprender-serve/src/gguf/spm_bpe.rs new file mode 100644 index 0000000000..d1df6bf662 --- /dev/null +++ b/crates/aprender-serve/src/gguf/spm_bpe.rs @@ -0,0 +1,338 @@ +//! Canonical SPM-style BPE encoding for GGUF `tokenizer.ggml.model = "gemma4"` vocabularies +//! (APR-EMBED-001 EG-1). +//! +//! EmbeddingGemma 2 ships a 262,144-token `gemma4` vocabulary: ranked merges over raw UTF-8, +//! with SentencePiece's `▁` (U+2581) standing for a space and `<0xXX>` byte tokens as the +//! fallback. `GGUFModel::encode` used to run greedy longest-match on it, which is not the +//! model's tokenization, and `apr tokenize` said so with `greedy-fallback`. +//! +//! This module ports the reference implementation, llama.cpp (the pinned comparator, +//! `scripts/llama_pin.toml`), `LLAMA_VOCAB_PRE_TYPE_GEMMA4` in `src/llama-vocab.cpp`: +//! - special tokens are split out first, longest first (`tokenizer_st_partition`, shared with +//! [`super::byte_level_bpe`]); +//! - every space of a raw fragment becomes `▁` (`llama_escape_whitespace`), with no space prefix; +//! - the fragment splits into runs of newlines and runs of everything else (`[^\n]+|[\n]+`); +//! - a newline run that is itself a token is one symbol (llama.cpp PR 21343); +//! - otherwise each UTF-8 character is a symbol, and merges apply by TEXT, lowest rank first, +//! leftmost on ties, skipping a pair whose sides changed since it was queued; +//! - a final symbol that is not a token is spelled byte by byte as `<0xXX>` tokens, and a byte +//! with no such token is dropped, as llama.cpp drops it. + +use std::collections::hash_map::DefaultHasher; +use std::collections::{BinaryHeap, HashMap}; +use std::hash::{Hash, Hasher}; +use std::sync::{Arc, Mutex, OnceLock}; + +use super::byte_level_bpe::{partition_specials, special_tokens, token_types, Fragment}; +use super::types::GGUFValue; + +/// The `tokenizer.ggml.model` values this module encodes. +const SPM_BPE_MODELS: [&str; 1] = ["gemma4"]; + +/// SentencePiece's visible space. +const SPM_SPACE: char = '\u{2581}'; + +/// Is this file's vocabulary an SPM-style BPE vocabulary (`tokenizer.ggml.model = "gemma4"`)? +#[must_use] +pub fn is_spm_bpe(metadata: &HashMap) -> bool { + matches!( + metadata.get("tokenizer.ggml.model"), + Some(GGUFValue::String(s)) if SPM_BPE_MODELS.contains(&s.as_str()) + ) +} + +/// Why a GGUF vocabulary cannot be encoded by [`SpmBpe`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpmBpeRefusal { + /// `tokenizer.ggml.model` is absent or names another tokenizer. + NotSpmBpe(String), + /// The file has no `tokenizer.ggml.merges` (llama.cpp refuses such a file too). + MissingMerges, +} + +impl std::fmt::Display for SpmBpeRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotSpmBpe(m) => write!(f, "tokenizer.ggml.model '{m}' is not SPM-style BPE"), + Self::MissingMerges => write!(f, "the file has no tokenizer.ggml.merges"), + } + } +} + +/// An SPM-style BPE tokenizer built from a GGUF's vocabulary, merges and token types. +#[derive(Debug)] +pub struct SpmBpe { + token_to_id: HashMap, + /// `(first, second) -> rank`, from `tokenizer.ggml.merges` in file order; first rule wins. + ranks: HashMap<(String, String), u32>, + /// The `<0xXX>` byte tokens' ids, by byte. + byte_ids: [Option; 256], + /// Special tokens, longest text first. + specials: Vec<(String, u32)>, +} + +impl SpmBpe { + /// Build the tokenizer for a `gemma4`-model GGUF, cached per distinct (vocabulary, merges, + /// token types) as [`super::byte_level_bpe::ByteLevelBpe::from_gguf`] is. + /// + /// # Errors + /// [`SpmBpeRefusal`] when the file is not SPM-style BPE or lacks merges. + pub fn from_gguf( + metadata: &HashMap, + vocab: &[String], + ) -> Result, SpmBpeRefusal> { + if !is_spm_bpe(metadata) { + let model = match metadata.get("tokenizer.ggml.model") { + Some(GGUFValue::String(s)) => s.clone(), + _ => "absent".to_string(), + }; + return Err(SpmBpeRefusal::NotSpmBpe(model)); + } + let merges: Vec<&str> = match metadata.get("tokenizer.ggml.merges") { + Some(GGUFValue::Array(values)) => values + .iter() + .filter_map(|v| match v { + GGUFValue::String(s) => Some(s.as_str()), + _ => None, + }) + .collect(), + _ => return Err(SpmBpeRefusal::MissingMerges), + }; + let token_types = token_types(metadata); + + let mut hasher = DefaultHasher::new(); + vocab.hash(&mut hasher); + merges.hash(&mut hasher); + token_types.hash(&mut hasher); + let key = hasher.finish(); + + static CACHE: OnceLock)>>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(Vec::new())); + if let Some(hit) = cache.lock().ok().and_then(|c| { + c.iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| Arc::clone(v)) + }) { + return Ok(hit); + } + let built = Arc::new(Self::build(vocab, &merges, &token_types)); + if let Ok(mut c) = cache.lock() { + if c.len() >= 8 { + c.remove(0); + } + c.push((key, Arc::clone(&built))); + } + Ok(built) + } + + /// Build from explicit tables (no cache). + #[must_use] + pub fn build(vocab: &[String], merges: &[&str], token_types: &[i32]) -> Self { + let token_to_id: HashMap = vocab + .iter() + .enumerate() + .map(|(id, t)| (t.clone(), id as u32)) + .collect(); + + let mut ranks = HashMap::with_capacity(merges.len()); + for (rank, rule) in merges.iter().enumerate() { + // llama.cpp splits at the first space at or after BYTE 1 (`find(' ', 1)`), so a rule + // may merge a first piece that is itself a space; a rule with no such space is + // ("", ""). A space is ASCII, so the split point is always a char boundary. + let split = rule.bytes().skip(1).position(|b| b == b' ').map(|p| p + 1); + let (first, second) = match split { + Some(at) => (&rule[..at], &rule[at + 1..]), + None => ("", ""), + }; + ranks + .entry((first.to_string(), second.to_string())) + .or_insert(rank as u32); + } + + let mut byte_ids = [None; 256]; + for (b, id) in byte_ids.iter_mut().enumerate() { + *id = token_to_id.get(byte_token(b as u8).as_str()).copied(); + } + + Self { + token_to_id, + ranks, + byte_ids, + specials: special_tokens(vocab, token_types), + } + } + + /// Encode `text` into token ids. Special tokens in the text are matched first, as + /// llama.cpp does with `parse_special = true`; no BOS or EOS is added (the caller owns that). + #[must_use] + pub fn encode(&self, text: &str) -> Vec { + let mut ids = Vec::new(); + for fragment in partition_specials(text, &self.specials) { + match fragment { + Fragment::Special(id) => ids.push(id), + Fragment::Raw(raw) => { + let escaped = raw.replace(' ', &SPM_SPACE.to_string()); + for word in newline_runs(&escaped) { + self.encode_word(word, &mut ids); + } + }, + } + } + ids + } + + /// BPE over one newline-free (or newline-only) word: a port of + /// `llm_tokenizer_bpe_session::tokenize` for `LLAMA_VOCAB_PRE_TYPE_GEMMA4`. + fn encode_word(&self, word: &str, out: &mut Vec) { + let mut syms = if word.bytes().all(|b| b == b'\n') && self.token_to_id.contains_key(word) { + vec![Sym { + start: 0, + len: word.len(), + prev: None, + next: None, + }] + } else { + seed_symbols(word) + }; + self.merge_all(word, &mut syms); + let mut at = (!syms.is_empty()).then_some(0); + while let Some(i) = at { + let s = syms[i]; + if s.len > 0 { + self.emit(&word[s.start..s.start + s.len], out); + } + at = s.next; + } + } + + /// The merge loop: pop the best pending pair, apply it if both sides still spell what they + /// spelled when it was queued, and queue the two pairs it creates with its neighbours. + fn merge_all(&self, word: &str, syms: &mut [Sym]) { + let mut queue = BinaryHeap::new(); + for left in 1..syms.len() { + self.enqueue(word, syms, left - 1, left, &mut queue); + } + while let Some(p) = queue.pop() { + let (l, r) = (syms[p.left], syms[p.right]); + if l.len == 0 || r.len == 0 || l.len + r.len != p.len || l.start + l.len != r.start { + continue; + } + if word[l.start..l.start + p.len] != *p.text { + continue; + } + syms[p.left].len += r.len; + syms[p.right].len = 0; + syms[p.left].next = r.next; + if let Some(nn) = r.next { + syms[nn].prev = Some(p.left); + } + if let Some(prev) = syms[p.left].prev { + self.enqueue(word, syms, prev, p.left, &mut queue); + } + if let Some(nn) = syms[p.left].next { + self.enqueue(word, syms, p.left, nn, &mut queue); + } + } + } + + fn enqueue( + &self, + word: &str, + syms: &[Sym], + left: usize, + right: usize, + queue: &mut BinaryHeap, + ) { + let (l, r) = (syms[left], syms[right]); + let first = &word[l.start..l.start + l.len]; + let second = &word[r.start..r.start + r.len]; + if let Some(&rank) = self.ranks.get(&(first.to_string(), second.to_string())) { + queue.push(Pending { + rank, + left, + right, + len: l.len + r.len, + text: format!("{first}{second}"), + }); + } + } + + /// One final symbol: its token, or its bytes as `<0xXX>` tokens. + fn emit(&self, piece: &str, out: &mut Vec) { + if let Some(&id) = self.token_to_id.get(piece) { + out.push(id); + return; + } + out.extend(piece.bytes().filter_map(|b| self.byte_ids[usize::from(b)])); + } +} + +/// The SentencePiece byte token for `b`: `<0xXX>`, upper-case hex. +fn byte_token(b: u8) -> String { + format!("<0x{b:02X}>") +} + +/// `[^\n]+|[\n]+`: the fragment as alternating runs of newlines and of everything else. +fn newline_runs(text: &str) -> Vec<&str> { + let mut runs = Vec::new(); + let mut start = 0; + let bytes = text.as_bytes(); + for i in 1..=bytes.len() { + if i == bytes.len() || (bytes[i] == b'\n') != (bytes[start] == b'\n') { + runs.push(&text[start..i]); + start = i; + } + } + runs +} + +/// One symbol per UTF-8 character. +fn seed_symbols(word: &str) -> Vec { + let n = word.chars().count(); + word.char_indices() + .enumerate() + .map(|(i, (start, c))| Sym { + start, + len: c.len_utf8(), + prev: i.checked_sub(1), + next: Some(i + 1).filter(|&j| j < n), + }) + .collect() +} + +/// A symbol: a byte range of the word, linked to its live neighbours. `len == 0` once merged +/// into its left neighbour. +#[derive(Debug, Clone, Copy)] +struct Sym { + start: usize, + len: usize, + prev: Option, + next: Option, +} + +/// A candidate merge. Ordered so that `BinaryHeap` (a max-heap) pops the LOWEST rank first +/// and, among equal ranks, the LEFTMOST pair: llama.cpp's `llm_bigram_bpe::comparator`. +#[derive(Debug, PartialEq, Eq)] +struct Pending { + rank: u32, + left: usize, + right: usize, + len: usize, + text: String, +} + +impl Ord for Pending { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.rank.cmp(&self.rank).then(other.left.cmp(&self.left)) + } +} + +impl PartialOrd for Pending { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(test)] +#[path = "spm_bpe_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/gguf/spm_bpe_tests.rs b/crates/aprender-serve/src/gguf/spm_bpe_tests.rs new file mode 100644 index 0000000000..1133e7d3f3 --- /dev/null +++ b/crates/aprender-serve/src/gguf/spm_bpe_tests.rs @@ -0,0 +1,218 @@ +//! APR-EMBED-001 EG-1: case tables for the SPM-style BPE port. Each row is worked through +//! llama.cpp's `llm_tokenizer_bpe_session::tokenize` for `LLAMA_VOCAB_PRE_TYPE_GEMMA4` by hand; +//! the end-to-end id parity against the pinned llama.cpp on the real files is +//! `scripts/tokenizer_parity.sh`. + +use super::*; + +const NORMAL: i32 = 1; +const CONTROL: i32 = 3; +const BYTE: i32 = 6; + +/// A tokenizer over `tokens` (all NORMAL, except `<0xXX>` as BYTE and `<...>` as CONTROL). +fn bpe(tokens: &[&str], merges: &[&str]) -> (SpmBpe, Vec) { + let vocab: Vec = tokens.iter().map(|t| (*t).to_string()).collect(); + let types: Vec = vocab + .iter() + .map(|t| { + if t.starts_with("<0x") { + BYTE + } else if t.starts_with('<') && t.ends_with('>') { + CONTROL + } else { + NORMAL + } + }) + .collect(); + (SpmBpe::build(&vocab, merges, &types), vocab) +} + +/// The token texts `text` encodes to. +fn pieces(tok: &(SpmBpe, Vec), text: &str) -> Vec { + tok.0 + .encode(text) + .into_iter() + .map(|id| tok.1[id as usize].clone()) + .collect() +} + +#[test] +fn merges_apply_lowest_rank_first() { + let tok = bpe(&["a", "b", "c", "ab", "bc", "abc"], &["b c", "a b", "a bc"]); + assert_eq!(pieces(&tok, "abc"), ["abc"], "b c (0), then a bc (2)"); + let tok = bpe(&["a", "b", "c", "ab", "bc", "abc"], &["b c", "a b"]); + assert_eq!( + pieces(&tok, "abc"), + ["a", "bc"], + "b c wins over a b; no a bc rule, so the word stops there" + ); + let tok = bpe(&["a", "b", "c", "ab", "bc", "abc"], &["a b", "b c"]); + assert_eq!(pieces(&tok, "abc"), ["ab", "c"], "a b wins over b c"); +} + +#[test] +fn equal_ranks_merge_leftmost_and_stale_pairs_are_skipped() { + let tok = bpe(&["a", "aa"], &["a a"]); + assert_eq!( + pieces(&tok, "aaa"), + ["aa", "a"], + "the leftmost pair merges first" + ); + assert_eq!( + pieces(&tok, "aaaa"), + ["aa", "aa"], + "the middle pair went stale when its left side merged" + ); +} + +#[test] +fn merges_are_keyed_by_text_not_by_vocabulary() { + let tok = bpe(&["a", "b", "ab"], &[]); + assert_eq!( + pieces(&tok, "ab"), + ["a", "b"], + "a token with no merge rule is never formed" + ); +} + +#[test] +fn spaces_become_the_visible_space_with_no_prefix() { + let tok = bpe(&["a", "b", "\u{2581}", "\u{2581}b"], &["\u{2581} b"]); + assert_eq!(pieces(&tok, "a b"), ["a", "\u{2581}b"]); + assert_eq!( + pieces(&tok, "b"), + ["b"], + "no space is prepended to the first word" + ); + assert_eq!(pieces(&tok, " "), ["\u{2581}"]); +} + +#[test] +fn merge_rules_split_at_the_first_space_after_byte_zero() { + let (tok, _) = bpe(&[], &["\u{2581} b", " x", " x y", "ab", "a b", "a b"]); + let rank = |a: &str, b: &str| tok.ranks.get(&(a.to_string(), b.to_string())).copied(); + assert_eq!(rank("\u{2581}", "b"), Some(0), "a multibyte first piece"); + assert_eq!( + rank(" ", "x"), + Some(1), + "a first piece that is itself a space" + ); + assert_eq!(rank(" x", "y"), Some(2)); + assert_eq!( + rank("", ""), + Some(3), + "a rule with no space is (\"\", \"\")" + ); + assert_eq!( + rank("a", "b"), + Some(4), + "a duplicate rule keeps its first rank" + ); +} + +#[test] +fn newline_runs_split_the_fragment() { + assert_eq!(newline_runs("a\n\nb c\n"), ["a", "\n\n", "b c", "\n"]); + assert_eq!(newline_runs("\n"), ["\n"]); + assert!(newline_runs("").is_empty()); +} + +#[test] +fn a_newline_run_that_is_a_token_is_one_symbol() { + let tok = bpe(&["a", "b", "\n", "\n\n"], &[]); + assert_eq!( + pieces(&tok, "a\n\nb"), + ["a", "\n\n", "b"], + "no merge rule needed" + ); + let tok = bpe(&["a", "\n"], &[]); + assert_eq!( + pieces(&tok, "a\n\n"), + ["a", "\n", "\n"], + "not a token: per char" + ); +} + +#[test] +fn words_never_merge_across_a_newline() { + let tok = bpe(&["a", "\n", "a\n"], &["a \n"]); + assert_eq!(pieces(&tok, "a\n"), ["a", "\n"]); +} + +#[test] +fn unknown_symbols_fall_back_to_upper_case_byte_tokens() { + let tok = bpe(&["a", "<0xC3>", "<0xA9>"], &[]); + assert_eq!(pieces(&tok, "a\u{e9}"), ["a", "<0xC3>", "<0xA9>"]); + let tok = bpe(&["a", "<0xC3>"], &[]); + assert_eq!( + pieces(&tok, "\u{e9}a"), + ["<0xC3>", "a"], + "a byte with no token is dropped" + ); +} + +#[test] +fn special_tokens_split_out_first_and_never_merge() { + let tok = bpe( + &["", "", "", "a", "\u{2581}", "\u{2581}a"], + &["\u{2581} a"], + ); + assert_eq!( + pieces(&tok, " a"), + ["", "\u{2581}a", ""] + ); + assert!(tok.0.encode("").is_empty()); +} + +fn metadata(model: Option<&str>, merges: Option<&[&str]>) -> HashMap { + let mut m = HashMap::new(); + if let Some(model) = model { + m.insert( + "tokenizer.ggml.model".to_string(), + GGUFValue::String(model.to_string()), + ); + } + if let Some(merges) = merges { + m.insert( + "tokenizer.ggml.merges".to_string(), + GGUFValue::Array( + merges + .iter() + .map(|r| GGUFValue::String((*r).to_string())) + .collect(), + ), + ); + } + m +} + +#[test] +fn only_gemma4_is_spm_bpe() { + assert!(is_spm_bpe(&metadata(Some("gemma4"), None))); + for other in ["llama", "gpt2", "bpe", "Gemma4", "gemma"] { + assert!(!is_spm_bpe(&metadata(Some(other), None)), "{other}"); + } + assert!(!is_spm_bpe(&metadata(None, None))); +} + +#[test] +fn from_gguf_refuses_by_name_and_caches_by_tables() { + let vocab: Vec = ["a", "b", "ab"].iter().map(|t| (*t).to_string()).collect(); + assert_eq!( + SpmBpe::from_gguf(&metadata(Some("llama"), Some(&["a b"])), &vocab).err(), + Some(SpmBpeRefusal::NotSpmBpe("llama".to_string())) + ); + assert_eq!( + SpmBpe::from_gguf(&metadata(None, None), &vocab).err(), + Some(SpmBpeRefusal::NotSpmBpe("absent".to_string())) + ); + assert_eq!( + SpmBpe::from_gguf(&metadata(Some("gemma4"), None), &vocab).err(), + Some(SpmBpeRefusal::MissingMerges) + ); + let md = metadata(Some("gemma4"), Some(&["a b"])); + let first = SpmBpe::from_gguf(&md, &vocab).expect("gemma4 with merges"); + let again = SpmBpe::from_gguf(&md, &vocab).expect("gemma4 with merges"); + assert!(Arc::ptr_eq(&first, &again), "the same tables hit the cache"); + assert_eq!(first.encode("ab"), [2]); +} diff --git a/crates/aprender-serve/src/gguf/token.rs b/crates/aprender-serve/src/gguf/token.rs index 78182f2184..77d2da73fa 100644 --- a/crates/aprender-serve/src/gguf/token.rs +++ b/crates/aprender-serve/src/gguf/token.rs @@ -171,7 +171,20 @@ impl GGUFModel { return None; } crate::gguf::byte_level_bpe::ByteLevelBpe::from_gguf(&self.metadata, vocab) - .map_err(|refusal| warn_greedy_tokenizer_fallback_once(&refusal)) + .map_err(|refusal| warn_greedy_tokenizer_fallback_once("byte-level BPE", &refusal)) + .ok() + .map(|bpe| bpe.encode(text)) + } + + /// APR-EMBED-001 EG-1: the canonical encoding of an SPM-style BPE (`gemma4`) vocabulary, or + /// `None` when the file is not one. A `gemma4` file without merges is refused by llama.cpp; + /// here it says so once on stderr and takes the greedy fallback. + fn encode_spm_bpe(&self, text: &str, vocab: &[String]) -> Option> { + if !crate::gguf::spm_bpe::is_spm_bpe(&self.metadata) { + return None; + } + crate::gguf::spm_bpe::SpmBpe::from_gguf(&self.metadata, vocab) + .map_err(|refusal| warn_greedy_tokenizer_fallback_once("SPM-style BPE", &refusal)) .ok() .map(|bpe| bpe.encode(text)) } @@ -192,10 +205,16 @@ impl GGUFModel { pub fn encode(&self, text: &str) -> Option> { let vocab = self.vocabulary()?; - if let Some(ids) = self.encode_byte_level(text, &vocab) { - return Some(ids); - } + let ids = self + .encode_spm_bpe(text, &vocab) + .or_else(|| self.encode_byte_level(text, &vocab)) + .unwrap_or_else(|| self.encode_greedy(text, &vocab)); + Some(ids) + } + /// The greedy longest-match fallback of [`Self::encode`], for a vocabulary no canonical + /// path covers. NOT the model's tokenization. + fn encode_greedy(&self, text: &str, vocab: &[String]) -> Vec { // Build reverse lookup: token string -> token ID let token_to_id: std::collections::HashMap<&str, u32> = vocab .iter() @@ -209,7 +228,7 @@ impl GGUFModel { // fallback when a file has no type table; it missed llama-family ``, so a // TinyLlama chat prompt encoded `.` as `.`. let special_tokens = crate::gguf::byte_level_bpe::special_tokens( - &vocab, + vocab, &crate::gguf::byte_level_bpe::token_types(&self.metadata), ); @@ -253,58 +272,10 @@ impl GGUFModel { text_with_prefix.replace(' ', &space_char.to_string()) }; - let mut remaining = processed.as_str(); - - while !remaining.is_empty() { - // Greedy longest match using character boundaries (not byte indices) - let mut best_byte_len = 0; - let mut best_id = None; - - // Collect character byte offsets for proper slicing - let char_indices: Vec = remaining - .char_indices() - .map(|(i, _)| i) - .chain(std::iter::once(remaining.len())) - .collect(); - - // Try all prefixes up to 32 chars (reasonable max token length) - for char_count in 1..=char_indices.len().saturating_sub(1).min(32) { - let byte_end = char_indices[char_count]; - let prefix = &remaining[..byte_end]; - if let Some(&id) = token_to_id.get(prefix) { - best_byte_len = byte_end; - best_id = Some(id); - } - } - - if let Some(id) = best_id { - tokens.push(id); - remaining = &remaining[best_byte_len..]; - } else { - // No match found - try single UTF-8 char as byte tokens - // SAFETY: remaining is non-empty (loop condition guarantees this) - let ch = remaining - .chars() - .next() - .expect("loop invariant: remaining non-empty"); - let ch_len = ch.len_utf8(); - - // Look for byte tokens like <0x48> for 'H' - for byte in remaining[..ch_len].bytes() { - let byte_token = format!("<0x{:02X}>", byte); - if let Some(&id) = token_to_id.get(byte_token.as_str()) { - tokens.push(id); - } else { - // Unknown byte - use a common unknown token ID (usually 0 or 1) - tokens.push(0); - } - } - remaining = &remaining[ch_len..]; - } - } + greedy_longest_match(&processed, &token_to_id, &mut tokens); } - Some(tokens) + tokens } /// The vocabulary as schema-constrained decoding sees it (#3568): each @@ -416,12 +387,69 @@ include!("loader_parse.rs"); include!("metadata.rs"); /// The greedy longest-match fallback is not the model's tokenization; say so, once (#3726). -fn warn_greedy_tokenizer_fallback_once(refusal: &crate::gguf::byte_level_bpe::ByteLevelBpeRefusal) { +fn warn_greedy_tokenizer_fallback_once(tokenizer: &str, refusal: &dyn std::fmt::Display) { static WARNED: std::sync::Once = std::sync::Once::new(); WARNED.call_once(|| { eprintln!( - "warning: byte-level BPE tokenizer: {refusal}; falling back to greedy longest-match, \ + "warning: {tokenizer} tokenizer: {refusal}; falling back to greedy longest-match, \ which does NOT reproduce the model's tokenization (#3726)" ); }); } + +/// Greedy longest match over one non-special segment, at most 32 characters per token; a +/// character no prefix matches is spelled as `<0xXX>` byte tokens, or id 0 for a byte with none. +fn greedy_longest_match( + processed: &str, + token_to_id: &std::collections::HashMap<&str, u32>, + tokens: &mut Vec, +) { + let mut remaining = processed; + while !remaining.is_empty() { + // Greedy longest match using character boundaries (not byte indices) + let mut best_byte_len = 0; + let mut best_id = None; + + // Collect character byte offsets for proper slicing + let char_indices: Vec = remaining + .char_indices() + .map(|(i, _)| i) + .chain(std::iter::once(remaining.len())) + .collect(); + + // Try all prefixes up to 32 chars (reasonable max token length) + for char_count in 1..=char_indices.len().saturating_sub(1).min(32) { + let byte_end = char_indices[char_count]; + let prefix = &remaining[..byte_end]; + if let Some(&id) = token_to_id.get(prefix) { + best_byte_len = byte_end; + best_id = Some(id); + } + } + + if let Some(id) = best_id { + tokens.push(id); + remaining = &remaining[best_byte_len..]; + } else { + // No match found - try single UTF-8 char as byte tokens + // SAFETY: remaining is non-empty (loop condition guarantees this) + let ch = remaining + .chars() + .next() + .expect("loop invariant: remaining non-empty"); + let ch_len = ch.len_utf8(); + + // Look for byte tokens like <0x48> for 'H' + for byte in remaining[..ch_len].bytes() { + let byte_token = format!("<0x{:02X}>", byte); + if let Some(&id) = token_to_id.get(byte_token.as_str()) { + tokens.push(id); + } else { + // Unknown byte - use a common unknown token ID (usually 0 or 1) + tokens.push(0); + } + } + remaining = &remaining[ch_len..]; + } + } +} diff --git a/scripts/tokenizer_parity.sh b/scripts/tokenizer_parity.sh index 0eca3c3918..6bf5326e3a 100644 --- a/scripts/tokenizer_parity.sh +++ b/scripts/tokenizer_parity.sh @@ -132,7 +132,7 @@ for m in "${models[@]}"; do path=${status%|*} roundtrip=${status##*|} case "$path" in - *"not a byte-level vocabulary"*) + *"not a byte-level or SPM-style BPE vocabulary"*) printf 'UNCOVERED %s %s %s\n' "$mname" "$fname" "$path" uncovered=$((uncovered + 1)) continue ;; diff --git a/scripts/tokenizer_parity_lib.sh b/scripts/tokenizer_parity_lib.sh index 9a18630ead..6ed4efa812 100755 --- a/scripts/tokenizer_parity_lib.sh +++ b/scripts/tokenizer_parity_lib.sh @@ -68,7 +68,7 @@ tp_apr_status() { printf '%s|%s' "$path" "$rt" } -# tp_is_canonical PATH -> status 0 iff the encoder took the canonical byte-level BPE path. +# tp_is_canonical PATH -> status 0 iff the encoder took a canonical path (byte-level BPE, or SPM-style BPE for gemma4). tp_is_canonical() { [ "${1-}" = "canonical" ] } From 54c969fa207decfb89d5e990d93392214930a33d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 07:33:35 +0200 Subject: [PATCH 09/21] docs(embed): EG-1 tokenizer parity receipt, 1120/1120 against a provisional oracle The pinned llama.cpp refuses gemma-embedding2 (D-3 is a STOP), so this records the provisional measurement, not the gate receipt. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- .../APR-EMBED-001/EG-1/tokenizer-parity.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/audits/APR-EMBED-001/EG-1/tokenizer-parity.md diff --git a/docs/audits/APR-EMBED-001/EG-1/tokenizer-parity.md b/docs/audits/APR-EMBED-001/EG-1/tokenizer-parity.md new file mode 100644 index 0000000000..85f51820a5 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-1/tokenizer-parity.md @@ -0,0 +1,20 @@ +# EG-1 tokenizer parity: provisional receipt (2026-10-08) + +**Result: 1120 / 1120 text-file pairs give identical token ids** (8 GGUF files × 140 texts). Every apr run took the +`canonical` path and round-tripped (`roundtrip: true`). + +| Item | Value | +|------|-------| +| apr | release build of `feat/apr-embed-001-eg1` @ 391318a784 (EG-1 slice 4), sha256 `106b0a678617fc78131e959113f2c54a1812285c04bc7c4b876305e70b232b68` | +| Oracle | llama.cpp b11490 (`9c2e0e491`) `llama-tokenize --ids --no-escape --no-bos`. **PROVISIONAL** (§7) | +| Pinned oracle | `scripts/llama_pin.toml` build_commit `d1d3c3396` refuses the file: `unknown model architecture: 'gemma-embedding2'`. Moving the pin is D-3, a §7 pin-change STOP, so it was not moved | +| Files | unsloth BF16, F16, Q8_0, UD-Q4_K_XL, UD-Q5_K_XL, UD-Q6_K_XL; ggml-org BF16, Q8_0 (text GGUFs; mmproj excluded) | +| Corpus | one file per row (last column) of `corpus/{batch,lengths,texts}.tsv`, `retrieval/{docs,queries}.tsv`, `finetune/heldout.tsv`, plus 3 edge files: double spaces, tabs, newline runs, accents, CJK, emoji, ``/`` specials, leading/trailing spaces, code with a trailing `\n\n\n` | + +## Why this is not the gate receipt + +`scripts/tokenizer_parity.sh` refuses any comparator other than the pinned commit, and it should. The comparison +above re-uses its library (`tp_llama_ids`, `tp_first_mismatch`, `tp_apr_status`) and its exact flags, but runs +against b11490. The comparator was checked to be live: `tp_first_mismatch "1 2 3" "1 9 3"` = 1 and +`tp_first_mismatch "1 2" "1 2 3"` = 2. The same files and corpus become the gate receipt once a pin that loads +`gemma-embedding2` is approved (D-3). From 765a03f00eafc5a10bd1aa01a38f196b4e687d8f Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 8 Oct 2026 10:31:39 +0200 Subject: [PATCH 10/21] chore(embed): EG-1 regenerate README contract count and roadmap aggregate embeddinggemma2-load-v1.yaml is a new contract; make readme-sync and make roadmap-aggregate keep FALSIFY-README-002 and the roadmap DRIFT guard green. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- README.md | 4 +- docs/roadmaps/roadmap.yaml | 180 +++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78a644977b..fcb1727312 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1895** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1897** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1895 contracts across inference, training, quantization, attention, FFN, +The tree carries 1897 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 7e7fb31a7b..293a2216b0 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24253,3 +24253,183 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:code notes: 'Cause: the test writes a script then execs it while sibling test threads fork; a child forked in that window holds the write fd, so exec fails with ETXTBSY (run 37403017673, workspace-test shard 3). Passed on the same PR head in run 37408602581.' +- id: PMAT-4899 + github_issue: 4899 + item_type: task + title: 'EG-0: Measure — probe, oracle, contracts, corpus (no engine code)' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:53:52Z + updated: 2026-10-07T17:53:52Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:measurement + notes: null + epic: 4898 +- id: PMAT-4900 + github_issue: 4900 + item_type: task + title: 'EG-1: Load — GGUF, tokenizer, kind' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:53:58Z + updated: 2026-10-07T17:53:58Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4901 + github_issue: 4901 + item_type: task + title: 'EG-2: Embed on CPU' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:04Z + updated: 2026-10-07T17:54:04Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4902 + github_issue: 4902 + item_type: task + title: 'EG-3: Serve' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:09Z + updated: 2026-10-07T17:54:09Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4903 + github_issue: 4903 + item_type: task + title: 'EG-4: CUDA on sm_89 and sm_121' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:14Z + updated: 2026-10-07T17:54:14Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4904 + github_issue: 4904 + item_type: task + title: 'EG-5: Refusals, apr qa, the ladder' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:19Z + updated: 2026-10-07T17:54:19Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4905 + github_issue: 4905 + item_type: task + title: 'EG-6: Model ops round trip' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:23Z + updated: 2026-10-07T17:54:23Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4906 + github_issue: 4906 + item_type: task + title: 'EG-7: Fine-tune' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:28Z + updated: 2026-10-07T17:54:28Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4907 + github_issue: 4907 + item_type: task + title: 'EG-8: Prune, then distill' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:33Z + updated: 2026-10-07T17:54:33Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 +- id: PMAT-4908 + github_issue: 4908 + item_type: task + title: 'EG-9: Evaluate, measure, dogfood' + status: planned + priority: high + assigned_to: null + created: 2026-10-07T17:54:38Z + updated: 2026-10-07T17:54:38Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 4898 From 85ac2d193b99f3bfd7ac6a5ca1d3715b1b318fc9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 9 Oct 2026 21:35:47 +0200 Subject: [PATCH 11/21] =?UTF-8?q?feat(embed):=20EG-1=20slice=205=20?= =?UTF-8?q?=E2=80=94=20apr=20flow=20draws=20an=20embedding=20model=20by=20?= =?UTF-8?q?kind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An embedding GGUF reaches `apr flow` with no LM head and per-layer attention widths. The decoder drawing assumed both, so flow now reads the model kind first and draws the embedding graph: token embedding, the per-layer input projection, the local and global blocks with their widths, the output projection and mean pooling. A decoder model takes the old path unchanged. Refs #4900 Co-Authored-By: Claude Opus 5.5 Agent: apr-embed --- crates/apr-cli/src/commands/flow.rs | 7 + crates/apr-cli/src/commands/flow_embedding.rs | 271 ++++++++++++++++++ .../src/commands/flow_embedding_tests.rs | 201 +++++++++++++ 3 files changed, 479 insertions(+) create mode 100644 crates/apr-cli/src/commands/flow_embedding.rs create mode 100644 crates/apr-cli/src/commands/flow_embedding_tests.rs diff --git a/crates/apr-cli/src/commands/flow.rs b/crates/apr-cli/src/commands/flow.rs index 1bf7d7a4dc..dcd232b6e5 100644 --- a/crates/apr-cli/src/commands/flow.rs +++ b/crates/apr-cli/src/commands/flow.rs @@ -69,6 +69,11 @@ pub(crate) fn run( .inspect(apr_path) .map_err(|e| CliError::InvalidFormat(format!("Failed to inspect: {e}")))?; + // APR-EMBED-001 EG-1: an embedding model is drawn by kind, never as a decoder with an LM head. + if let Some(embedding) = embedding_flow(&report)? { + return run_embedding_flow(apr_path, &report, &embedding, component, json_output); + } + let tensor_names: Vec = report.tensors.iter().map(|t| t.name.clone()).collect(); let arch = detect_architecture(&tensor_names); @@ -562,5 +567,7 @@ fn print_decoder_block(tensor_names: &[String], _verbose: bool) { } include!("flow_print_cross.rs"); +include!("flow_embedding.rs"); +include!("flow_embedding_tests.rs"); include!("flow_coverage_tests.rs"); include!("flow_03.rs"); diff --git a/crates/apr-cli/src/commands/flow_embedding.rs b/crates/apr-cli/src/commands/flow_embedding.rs new file mode 100644 index 0000000000..b69cba79fa --- /dev/null +++ b/crates/apr-cli/src/commands/flow_embedding.rs @@ -0,0 +1,271 @@ +// ════════════════════════════════════════════════════════════════════ +// APR-EMBED-001 EG-1: `apr flow` on an embedding model draws the embedding path by kind: +// bidirectional layers, pooling, projection, L2 normalisation. Never a decoder with an LM head +// and a next-token output (the EG-0 probe's silent wrong answer). +// Contract: contracts/embeddinggemma2-load-v1.yaml (model_kind, layer_attention_kind). +// ════════════════════════════════════════════════════════════════════ + +use realizar::model_kind::{layer_attention_kinds, model_kind, LayerAttention, ModelKind}; + +/// What `apr flow` draws for an embedding model. Every field is read from the file (I-9). +#[derive(Debug, Clone, PartialEq, Eq)] +struct EmbeddingFlow { + /// `general.architecture`, as the file carries it. + architecture: String, + /// `.embedding_length`. + hidden: usize, + /// One entry per layer, from the output width of `blk..attn_q.weight`. + layers: Vec, + /// `.attention.sliding_window`, the local layers' window. + sliding_window: Option, + /// A `blk.0.ffn_gate.weight` tensor is present. + gated_ffn: bool, + /// `.pooling_type`, named. + pooling: String, + /// `.embedding_length_out`, the projected width. + output_dim: Option, +} + +/// The embedding path of `report`, or `None` for a file that is not an embedding model. +/// +/// # Errors +/// An embedding architecture whose `attention.causal` flag contradicts it, a missing +/// `embedding_length` or `block_count`, or layer attention kinds that cannot be read. +fn embedding_flow( + report: &aprender::format::rosetta::InspectionReport, +) -> Result, CliError> { + let Some(arch) = report.metadata.get("general.architecture") else { + return Ok(None); + }; + let causal = report + .metadata + .get(&format!("{arch}.attention.causal")) + .map(|v| v == "true"); + if model_kind(arch, causal).map_err(CliError::InvalidFormat)? == ModelKind::Generative { + return Ok(None); + } + let required = |key: &str| { + meta_usize(report, arch, key) + .ok_or_else(|| CliError::InvalidFormat(format!("embedding model without {arch}.{key}"))) + }; + let hidden = required("embedding_length")?; + let widths = (0..required("block_count")?) + .map(|l| attn_q_width(report, l, hidden)) + .collect::, _>>()?; + Ok(Some(EmbeddingFlow { + architecture: arch.clone(), + hidden, + layers: layer_attention_kinds(&widths).map_err(CliError::InvalidFormat)?, + sliding_window: meta_usize(report, arch, "attention.sliding_window"), + gated_ffn: report + .tensors + .iter() + .any(|t| t.name == "blk.0.ffn_gate.weight"), + pooling: pooling_name(report.metadata.get(&format!("{arch}.pooling_type"))), + output_dim: meta_usize(report, arch, "embedding_length_out"), + })) +} + +/// `.` as an unsigned integer. +fn meta_usize( + report: &aprender::format::rosetta::InspectionReport, + arch: &str, + key: &str, +) -> Option { + report.metadata.get(&format!("{arch}.{key}"))?.parse().ok() +} + +/// The output width of layer `l`'s query projection: its element count over the model width. +/// Independent of the order the format lists the two dimensions in. +fn attn_q_width( + report: &aprender::format::rosetta::InspectionReport, + l: usize, + hidden: usize, +) -> Result { + let name = format!("blk.{l}.attn_q.weight"); + let numel: usize = report + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| CliError::InvalidFormat(format!("embedding model without {name}")))? + .shape + .iter() + .product(); + if hidden == 0 || numel % hidden != 0 { + return Err(CliError::InvalidFormat(format!( + "{name} has {numel} elements, not a multiple of embedding_length {hidden}" + ))); + } + Ok(numel / hidden) +} + +/// The GGUF `pooling_type` value, named (llama.cpp's `llama_pooling_type`). +fn pooling_name(value: Option<&String>) -> String { + match value.map(String::as_str) { + None => "unspecified".to_string(), + Some("0") => "none".to_string(), + Some("1") => "mean".to_string(), + Some("2") => "cls".to_string(), + Some("3") => "last".to_string(), + Some("4") => "rank".to_string(), + Some(other) => format!("pooling_type {other}"), + } +} + +/// Indices of the global-attention layers. +fn global_layers(flow: &EmbeddingFlow) -> Vec { + flow.layers + .iter() + .enumerate() + .filter(|(_, k)| **k == LayerAttention::Global) + .map(|(l, _)| l) + .collect() +} + +/// The embedding path as plain lines, one stage per box. +fn embedding_flow_lines(flow: &EmbeddingFlow) -> Vec { + let global = global_layers(flow); + let local = flow.layers.len() - global.len(); + let window = flow + .sliding_window + .map_or_else(String::new, |w| format!(", window {w}")); + let global_list = global + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "); + let ffn = if flow.gated_ffn { + "gated GELU FFN" + } else { + "GELU FFN" + }; + let out = flow + .output_dim + .map_or_else(|| "out_dim".to_string(), |d| d.to_string()); + let stages: [(String, Vec); 6] = [ + ( + "Token Embed".to_string(), + vec![format!( + "token_embd.weight → [batch, seq_len, {}]", + flow.hidden + )], + ), + ( + format!( + "Layers × {} (bidirectional, no causal mask)", + flow.layers.len() + ), + vec![ + format!( + "local × {local}{window}; global × {}: {global_list}", + global.len() + ), + "RMSNorm → self-attention → + residual".to_string(), + format!("RMSNorm → {ffn} → + residual"), + ], + ), + ("Final RMSNorm".to_string(), vec![]), + ( + format!("Pool: {}", flow.pooling), + vec![format!("over real tokens → [batch, {}]", flow.hidden)], + ), + ( + "Projection".to_string(), + vec![format!("[batch, {}] → [batch, {out}]", flow.hidden)], + ), + ("L2 normalise".to_string(), vec![]), + ]; + let mut lines = vec!["INPUT tokens [batch, seq_len]".to_string()]; + for (title, body) in &stages { + lines.extend([" │".to_string(), " ▼".to_string()]); + lines.extend(boxed(title, body)); + } + lines.extend([ + " │".to_string(), + " ▼".to_string(), + format!("OUTPUT embedding [batch, {out}]"), + ]); + lines +} + +/// One stage in a box sized to its widest line. +fn boxed(title: &str, body: &[String]) -> Vec { + let rows: Vec<&str> = std::iter::once(title) + .chain(body.iter().map(String::as_str)) + .collect(); + let width = rows.iter().map(|r| r.chars().count()).max().unwrap_or(0); + let mut out = vec![format!("┌{}┐", "─".repeat(width + 2))]; + for row in rows { + let pad = width - row.chars().count(); + out.push(format!("│ {row}{} │", " ".repeat(pad))); + } + out.push(format!("└{}┘", "─".repeat(width + 2))); + out +} + +/// The embedding path as JSON. +fn embedding_flow_json( + path: &Path, + report: &aprender::format::rosetta::InspectionReport, + flow: &EmbeddingFlow, +) -> serde_json::Value { + serde_json::json!({ + "file": path.file_name().and_then(|s| s.to_str()).unwrap_or("model"), + "format": report.format.to_string(), + "architecture": flow.architecture, + "kind": "embedding", + "component": "full", + "total_tensors": report.tensors.len(), + "hidden": flow.hidden, + "layers": flow.layers.len(), + "global_layers": global_layers(flow), + "sliding_window": flow.sliding_window, + "gated_ffn": flow.gated_ffn, + "pooling": flow.pooling, + "output_dim": flow.output_dim, + "normalize": "l2", + }) +} + +/// Draw an embedding model whole. Only `--component full` applies: it has no decoder, encoder +/// block or cross-attention, and drawing one would be a wrong answer. +fn run_embedding_flow( + path: &Path, + report: &aprender::format::rosetta::InspectionReport, + flow: &EmbeddingFlow, + component: FlowComponent, + json_output: bool, +) -> Result<(), CliError> { + if component != FlowComponent::Full { + return Err(CliError::InvalidInput(format!( + "'{}' is an embedding model (ModelKind::Embedding): `apr flow` draws it whole; \ + --component {component:?} applies to generative and encoder-decoder models. \ + Use --component full.", + flow.architecture + ))); + } + if json_output { + let json = embedding_flow_json(path, report, flow); + println!( + "{}", + serde_json::to_string_pretty(&json).unwrap_or_else(|_| json.to_string()) + ); + return Ok(()); + } + println!( + "{} {} ({}, {}, {})", + "Model:".bold(), + path.file_name() + .and_then(|s| s.to_str()) + .unwrap_or("model") + .cyan(), + "embedding".yellow(), + flow.architecture, + format!("{}", report.format).cyan() + ); + println!(); + for line in embedding_flow_lines(flow) { + println!("{line}"); + } + Ok(()) +} diff --git a/crates/apr-cli/src/commands/flow_embedding_tests.rs b/crates/apr-cli/src/commands/flow_embedding_tests.rs new file mode 100644 index 0000000000..fde1a7fd25 --- /dev/null +++ b/crates/apr-cli/src/commands/flow_embedding_tests.rs @@ -0,0 +1,201 @@ +// APR-EMBED-001 EG-1: `apr flow` draws an embedding model by kind (flow_embedding.rs). + +#[cfg(test)] +mod flow_embedding_tests { + use super::*; + use aprender::format::rosetta::{FormatType, InspectionReport, TensorInfo}; + use std::collections::BTreeMap; + + fn tensor(name: &str, shape: &[usize]) -> TensorInfo { + TensorInfo { + name: name.to_string(), + dtype: "F32".to_string(), + shape: shape.to_vec(), + size_bytes: 0, + stats: None, + } + } + + /// A 4-layer embedding report: layers 0..=2 local (attn_q 512 → 1024), layer 3 global (→ 2048). + fn embedding_report(arch: &str, q_out: [usize; 4]) -> InspectionReport { + let mut metadata = BTreeMap::new(); + for (k, v) in [ + ("general.architecture".to_string(), arch.to_string()), + (format!("{arch}.attention.causal"), "false".to_string()), + (format!("{arch}.embedding_length"), "512".to_string()), + (format!("{arch}.block_count"), "4".to_string()), + ( + format!("{arch}.attention.sliding_window"), + "1024".to_string(), + ), + (format!("{arch}.pooling_type"), "1".to_string()), + (format!("{arch}.embedding_length_out"), "768".to_string()), + ] { + metadata.insert(k, v); + } + let mut tensors = vec![ + tensor("token_embd.weight", &[512, 262_144]), + tensor("output.weight", &[512, 768]), + tensor("blk.0.ffn_gate.weight", &[512, 2048]), + ]; + for (l, w) in q_out.iter().enumerate() { + tensors.push(tensor(&format!("blk.{l}.attn_q.weight"), &[512, *w])); + } + InspectionReport { + format: FormatType::Gguf, + file_size: 0, + metadata, + tensors, + total_params: 0, + quantization: None, + architecture: Some(arch.to_string()), + } + } + + const LOCAL_LOCAL_LOCAL_GLOBAL: [usize; 4] = [1024, 1024, 1024, 2048]; + + #[test] + fn embedding_flow_reads_kind_layers_pooling_and_width_from_the_file_eg1() { + let flow = embedding_flow(&embedding_report( + "gemma-embedding2", + LOCAL_LOCAL_LOCAL_GLOBAL, + )) + .expect("admitted") + .expect("an embedding model"); + assert_eq!(flow.hidden, 512); + assert_eq!(global_layers(&flow), vec![3]); + assert_eq!(flow.sliding_window, Some(1024)); + assert!(flow.gated_ffn); + assert_eq!(flow.pooling, "mean"); + assert_eq!(flow.output_dim, Some(768)); + } + + /// I-9: the kinds follow the shapes, never the layer index. + #[test] + fn swapping_two_layers_attn_q_shapes_swaps_their_kinds_eg1() { + let flow = embedding_flow(&embedding_report( + "gemma-embedding2", + [2048, 1024, 1024, 1024], + )) + .expect("admitted") + .expect("an embedding model"); + assert_eq!(global_layers(&flow), vec![0]); + } + + /// The `gemma` prefix match is not widened, and a generative file keeps the generative flow. + #[test] + fn generative_and_other_gemma_architectures_are_not_drawn_as_embedding_eg1() { + for arch in ["qwen2", "gemma3", "gemma-embedding"] { + let got = embedding_flow(&embedding_report(arch, LOCAL_LOCAL_LOCAL_GLOBAL)) + .expect("not refused"); + assert!(got.is_none(), "{arch}"); + } + let mut no_arch = embedding_report("gemma-embedding2", LOCAL_LOCAL_LOCAL_GLOBAL); + no_arch.metadata.remove("general.architecture"); + assert!(embedding_flow(&no_arch).expect("not refused").is_none()); + } + + #[test] + fn embedding_architecture_with_causal_true_is_refused_eg1() { + let mut report = embedding_report("gemma-embedding2", LOCAL_LOCAL_LOCAL_GLOBAL); + report.metadata.insert( + "gemma-embedding2.attention.causal".to_string(), + "true".to_string(), + ); + assert!(embedding_flow(&report).is_err()); + } + + #[test] + fn single_attn_q_width_or_missing_layer_is_refused_eg1() { + assert!(embedding_flow(&embedding_report("gemma-embedding2", [1024; 4])).is_err()); + let mut report = embedding_report("gemma-embedding2", LOCAL_LOCAL_LOCAL_GLOBAL); + report.tensors.retain(|t| t.name != "blk.2.attn_q.weight"); + assert!(embedding_flow(&report).is_err()); + } + + /// The EG-0 probe's silent wrong answer: an LM head and a next-token output. + #[test] + fn embedding_flow_lines_draw_pool_projection_and_l2_never_an_lm_head_eg1() { + let flow = embedding_flow(&embedding_report( + "gemma-embedding2", + LOCAL_LOCAL_LOCAL_GLOBAL, + )) + .expect("admitted") + .expect("an embedding model"); + let text = embedding_flow_lines(&flow).join("\n"); + for want in [ + "bidirectional", + "local × 3, window 1024; global × 1: 3", + "gated GELU FFN", + "Pool: mean", + "[batch, 512] → [batch, 768]", + "L2 normalise", + "OUTPUT embedding [batch, 768]", + ] { + assert!(text.contains(want), "missing {want:?} in\n{text}"); + } + for never in ["LM Head", "logits", "next_token", "decoder"] { + assert!(!text.contains(never), "{never:?} in\n{text}"); + } + } + + #[test] + fn boxed_pads_every_row_to_the_widest_eg1() { + let rows = boxed("ab", &["abcd".to_string()]); + let widths: Vec = rows.iter().map(|r| r.chars().count()).collect(); + assert_eq!(widths, vec![8, 8, 8, 8], "{rows:?}"); + } + + #[test] + fn embedding_flow_json_names_the_kind_and_global_layers_eg1() { + let report = embedding_report("gemma-embedding2", LOCAL_LOCAL_LOCAL_GLOBAL); + let flow = embedding_flow(&report) + .expect("admitted") + .expect("an embedding model"); + let json = embedding_flow_json(Path::new("/m/e.gguf"), &report, &flow); + assert_eq!(json["kind"], "embedding"); + assert_eq!(json["file"], "e.gguf"); + assert_eq!(json["global_layers"], serde_json::json!([3])); + assert_eq!(json["pooling"], "mean"); + assert_eq!(json["output_dim"], 768); + } + + #[test] + fn only_component_full_draws_an_embedding_model_eg1() { + let report = embedding_report("gemma-embedding2", LOCAL_LOCAL_LOCAL_GLOBAL); + let flow = embedding_flow(&report) + .expect("admitted") + .expect("an embedding model"); + for component in [ + FlowComponent::Encoder, + FlowComponent::Decoder, + FlowComponent::CrossAttention, + FlowComponent::SelfAttention, + FlowComponent::Ffn, + ] { + let err = run_embedding_flow(Path::new("e.gguf"), &report, &flow, component, true) + .expect_err("refused"); + assert!(err.to_string().contains("--component full"), "{err}"); + } + run_embedding_flow( + Path::new("e.gguf"), + &report, + &flow, + FlowComponent::Full, + true, + ) + .expect("full is drawn"); + } + + #[test] + fn pooling_names_follow_llama_pooling_type_eg1() { + let name = |v: &str| pooling_name(Some(&v.to_string())); + assert_eq!(pooling_name(None), "unspecified"); + assert_eq!(name("0"), "none"); + assert_eq!(name("1"), "mean"); + assert_eq!(name("2"), "cls"); + assert_eq!(name("3"), "last"); + assert_eq!(name("4"), "rank"); + assert_eq!(name("9"), "pooling_type 9"); + } +} From 2a42c3d99a81ae4a2fb532dc8287bd1d143b7b6e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 9 Oct 2026 22:35:36 +0200 Subject: [PATCH 12/21] fix(gguf): EG-1 read BF16 tensors as F32, exactly The EG-1 verb sweep ran `apr validate` on the six Unsloth EmbeddingGemma 2 files. It failed three of them: BF16 (219 tensors), Q8_0 and UD-Q6_K_XL (one tensor each, per_layer_model_proj.weight, which every file stores as BF16). The cause was that GgufReader::get_tensor_f32 sized ggml type 30 but had no arm for it, so it refused each tensor with "no dequantizer". BF16 is the top 16 bits of an F32, so it widens bit-exactly. F16 and BF16 now share one two-byte path (widen_half), and each keeps its own widening. A short data section is still refused by name, and the bounds check no longer adds without overflow checking. Widening scope: get_all_tensors_f32 (the GH-375 fallback of `apr import`, and `apr convert`) now reads BF16 too. Unlike the #3947 IQ decoders this widens and decodes nothing, so it invents no weight. A test pins it. Tests (reader_tests_ggml_sizes.rs): - BF16 widens bit-exactly, including -0.0, the smallest normal and infinity; the 0xEE tail is not read. - F16 keeps its own widening. 0x3C00 is 1.0 in F16 and 2^-7 in BF16, so a swapped widener fails. - Short BF16 data is refused. - The whole-model load returns the same exact values. Refs #4900 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- .../format/gguf/reader_tests_ggml_sizes.rs | 71 +++++++++++++++++++ crates/aprender-core/src/format/gguf/shape.rs | 59 +++++++++------ 2 files changed, 108 insertions(+), 22 deletions(-) diff --git a/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs index b95cc1972a..0d31559c7d 100644 --- a/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs +++ b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs @@ -261,3 +261,74 @@ fn test_3947_whole_model_f32_load_still_refuses_decoded_iq_types() { ); } } + +/// Values whose low 16 bits are zero, so BF16 holds each exactly: the sign of zero, the +/// smallest normal, a short fraction, a large negative value and infinity. +const BF16_EXACT: [f32; 8] = [ + 1.0, + -2.5, + 0.0, + -0.0, + f32::MIN_POSITIVE, + 3.140625, + -65280.0, + f32::INFINITY, +]; + +/// The BF16 bytes of `values`: the top 16 bits of each, little-endian. +fn bf16_bytes(values: &[f32]) -> Vec { + values + .iter() + .flat_map(|v| ((v.to_bits() >> 16) as u16).to_le_bytes()) + .collect() +} + +/// APR-EMBED-001 EG-1 (#4900): `get_tensor_f32` refused BF16 (ggml type 30) as having no +/// dequantizer, so `apr validate` failed every BF16 tensor of the EmbeddingGemma 2 files. +/// BF16 is the top half of an F32, so it widens bit-exactly; the 0xEE tail is not read. +#[test] +fn test_4900_get_tensor_f32_widens_bf16_bit_exactly() { + assert!( + BF16_EXACT.iter().all(|v| v.to_bits() & 0xFFFF == 0), + "fixture: every value must be exact in BF16" + ); + let n = BF16_EXACT.len(); + let reader = one_tensor(&[n as u64], 30, &bf16_bytes(&BF16_EXACT), 64); + let (got, shape) = reader.get_tensor_f32("t.weight").expect("BF16 must widen"); + assert_eq!(shape, vec![n]); + let bits = |v: &[f32]| v.iter().map(|x| x.to_bits()).collect::>(); + assert_eq!(bits(&got), bits(&BF16_EXACT), "BF16 must widen bit-exactly"); +} + +/// F16 shares the two-byte path with BF16 and keeps its own widening: 0x3C00 is 1.0 in +/// F16 and 2^-7 in BF16, so a swapped widener fails here. +#[test] +fn test_4900_f16_keeps_its_own_widening() { + let halves: [u16; 4] = [0x3C00, 0xC000, 0x7BFF, 0x0000]; + let bytes: Vec = halves.iter().flat_map(|h| h.to_le_bytes()).collect(); + let reader = one_tensor(&[4], 1, &bytes, 64); + let (got, _) = reader.get_tensor_f32("t.weight").expect("F16 widens"); + assert_eq!(got, vec![1.0, -2.0, 65504.0, 0.0]); +} + +/// A BF16 tensor whose data runs past the end of the file is refused by name, not read short. +#[test] +fn test_4900_short_bf16_data_is_refused() { + let reader = one_tensor(&[64], 30, &payload(64), 0); + let err = reader + .get_tensor_f32("t.weight") + .expect_err("64 elements need 128 bytes") + .to_string(); + assert!(err.contains("t.weight") && err.contains("exceeds file size"), "{err}"); +} + +/// The whole-model F32 load (`apr import`'s GH-375 fallback, `apr convert`) now reads BF16 +/// too, and gets the same exact values: unlike the #3947 IQ types, nothing is decoded, only +/// widened, so no weight is invented. +#[test] +fn test_4900_whole_model_f32_load_widens_bf16_exactly() { + let reader = one_tensor(&[BF16_EXACT.len() as u64], 30, &bf16_bytes(&BF16_EXACT), 0); + let all = reader.get_all_tensors_f32().expect("BF16 loads whole-model"); + let (got, _) = all.get("t.weight").expect("tensor present"); + assert!(got.iter().zip(&BF16_EXACT).all(|(a, b)| a.to_bits() == b.to_bits())); +} diff --git a/crates/aprender-core/src/format/gguf/shape.rs b/crates/aprender-core/src/format/gguf/shape.rs index d63c593d1f..af06bf93c1 100644 --- a/crates/aprender-core/src/format/gguf/shape.rs +++ b/crates/aprender-core/src/format/gguf/shape.rs @@ -62,28 +62,8 @@ impl GgufReader { .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) .collect() } - 1 => { - // F16 - convert to F32 - // BUG-GGUF-002 FIX: Use checked_mul for byte size calculation - let byte_size = - num_elements - .checked_mul(2) - .ok_or_else(|| AprenderError::FormatError { - message: format!("Tensor '{}' byte size calculation overflow", name), - })?; - if tensor_start + byte_size > self.data.len() { - return Err(AprenderError::FormatError { - message: format!("Tensor '{name}' data exceeds file size"), - }); - } - let bytes = &self.data[tensor_start..tensor_start + byte_size]; - bytes - .as_chunks::<2>() - .0 - .iter() - .map(|c| f16_to_f32(u16::from_le_bytes([c[0], c[1]]))) - .collect() - } + // F16 (1) and BF16 (30): two bytes per element, each widened exactly. + 1 | 30 => widen_half(&self.data, tensor_start, num_elements, name, meta.dtype)?, // GGML dtype values (from ggml.h): // 0=F32, 1=F16, 2=Q4_0, 3=Q4_1, 6=Q5_0, 7=Q5_1, 8=Q8_0, 9=Q8_1 // 10=Q2_K, 11=Q3_K, 12=Q4_K, 13=Q5_K, 14=Q6_K @@ -307,3 +287,38 @@ fn unsized_ggml_type_error(tensor: &str, e: &trueno_quant::GgmlTypeError) -> Apr ), } } + +/// F16 (ggml type 1) or BF16 (ggml type 30) data widened to F32. Both widen exactly: F16 +/// through `f16_to_f32`, BF16 by being the top 16 bits of an F32 (APR-EMBED-001 EG-1, #4900: +/// `apr validate` refused every BF16 tensor of the EmbeddingGemma 2 files). +fn widen_half( + data: &[u8], + start: usize, + num_elements: usize, + name: &str, + dtype: u32, +) -> Result> { + // BUG-GGUF-002 FIX: Use checked_mul for byte size calculation + let byte_size = num_elements + .checked_mul(2) + .ok_or_else(|| AprenderError::FormatError { + message: format!("Tensor '{name}' byte size calculation overflow"), + })?; + let bytes = start + .checked_add(byte_size) + .and_then(|end| data.get(start..end)) + .ok_or_else(|| AprenderError::FormatError { + message: format!("Tensor '{name}' data exceeds file size"), + })?; + let widen: fn(u16) -> f32 = if dtype == 30 { + |bits| f32::from_bits(u32::from(bits) << 16) + } else { + f16_to_f32 + }; + Ok(bytes + .as_chunks::<2>() + .0 + .iter() + .map(|c| widen(u16::from_le_bytes(*c))) + .collect()) +} From de03bbb402fec80fce736892b39666c90dc40b10 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 9 Oct 2026 22:51:15 +0200 Subject: [PATCH 13/21] test(gguf): EG-1 BF16 fixture check as trailing_zeros (clippy verbose_bit_mask) clippy --profile test on aprender-core flagged the low-16-bit mask in the BF16 widening test; trailing_zeros() >= 16 is the same predicate (0 gives 32). Refs #4900 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs index 0d31559c7d..6208c4c35c 100644 --- a/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs +++ b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs @@ -289,7 +289,7 @@ fn bf16_bytes(values: &[f32]) -> Vec { #[test] fn test_4900_get_tensor_f32_widens_bf16_bit_exactly() { assert!( - BF16_EXACT.iter().all(|v| v.to_bits() & 0xFFFF == 0), + BF16_EXACT.iter().all(|v| v.to_bits().trailing_zeros() >= 16), "fixture: every value must be exact in BF16" ); let n = BF16_EXACT.len(); From f112cf6bc761d3419a73637db1e4f30c1e6cab8e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 00:19:34 +0200 Subject: [PATCH 14/21] docs(embed): EG-1 verb sweep receipt, 84/84 on 6 of 6 files The 14 verbs run on the six unsloth embeddinggemma-2 GGUF files, each cell scored on exit code 0 and a positive check of its output. Run 1 (85ac2d193b) scored 75/84: 3 validate failures from BF16 tensors that aprender-core could not read (fixed in 2a42c3d99a) and 6 list checks that looked for a file name an hf:// single-file pull never prints. Run 2 (2a42c3d99a) scored 84/84 with the list/rm checks reading the cache path from the pull log, so rm can no longer pass vacuously. With the tokenizer parity receipt (1120/1120), this covers both halves of the EG-1 done_when. Refs #4900 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- .../APR-EMBED-001/EG-1/verb-sweep-cells.tsv | 85 ++++++++++++++ docs/audits/APR-EMBED-001/EG-1/verb-sweep.md | 105 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 docs/audits/APR-EMBED-001/EG-1/verb-sweep-cells.tsv create mode 100644 docs/audits/APR-EMBED-001/EG-1/verb-sweep.md diff --git a/docs/audits/APR-EMBED-001/EG-1/verb-sweep-cells.tsv b/docs/audits/APR-EMBED-001/EG-1/verb-sweep-cells.tsv new file mode 100644 index 0000000000..29b469b6ae --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-1/verb-sweep-cells.tsv @@ -0,0 +1,85 @@ +verb file rc verdict check +pull BF16 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list BF16 0 PASS rc 0; lists the pulled file by its cache path +rm BF16 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize BF16 0 PASS rc 0; roundtrip: true; no greedy fallback +capability BF16 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect BF16 0 PASS rc 0; architecture gemma-embedding2; causal false +debug BF16 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate BF16 0 PASS rc 0; VALID, 0 contract violations +lint BF16 0 PASS rc 0; lint passed, 0 errors +tensors BF16 0 PASS rc 0; tensor table with the file's tensors +diff BF16 0 PASS rc 0 vs embeddinggemma-2-F16.gguf +hex BF16 0 PASS rc 0; GGUF v3 header +tree BF16 0 PASS rc 0; root line with the tensor count +flow BF16 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding +pull F16 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list F16 0 PASS rc 0; lists the pulled file by its cache path +rm F16 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize F16 0 PASS rc 0; roundtrip: true; no greedy fallback +capability F16 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect F16 0 PASS rc 0; architecture gemma-embedding2; causal false +debug F16 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate F16 0 PASS rc 0; VALID, 0 contract violations +lint F16 0 PASS rc 0; lint passed, 0 errors +tensors F16 0 PASS rc 0; tensor table with the file's tensors +diff F16 0 PASS rc 0 vs embeddinggemma-2-BF16.gguf +hex F16 0 PASS rc 0; GGUF v3 header +tree F16 0 PASS rc 0; root line with the tensor count +flow F16 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding +pull Q8_0 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list Q8_0 0 PASS rc 0; lists the pulled file by its cache path +rm Q8_0 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize Q8_0 0 PASS rc 0; roundtrip: true; no greedy fallback +capability Q8_0 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect Q8_0 0 PASS rc 0; architecture gemma-embedding2; causal false +debug Q8_0 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate Q8_0 0 PASS rc 0; VALID, 0 contract violations +lint Q8_0 0 PASS rc 0; lint passed, 0 errors +tensors Q8_0 0 PASS rc 0; tensor table with the file's tensors +diff Q8_0 0 PASS rc 0 vs embeddinggemma-2-BF16.gguf +hex Q8_0 0 PASS rc 0; GGUF v3 header +tree Q8_0 0 PASS rc 0; root line with the tensor count +flow Q8_0 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding +pull UD-Q4_K_XL 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list UD-Q4_K_XL 0 PASS rc 0; lists the pulled file by its cache path +rm UD-Q4_K_XL 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize UD-Q4_K_XL 0 PASS rc 0; roundtrip: true; no greedy fallback +capability UD-Q4_K_XL 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect UD-Q4_K_XL 0 PASS rc 0; architecture gemma-embedding2; causal false +debug UD-Q4_K_XL 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate UD-Q4_K_XL 0 PASS rc 0; VALID, 0 contract violations +lint UD-Q4_K_XL 0 PASS rc 0; lint passed, 0 errors +tensors UD-Q4_K_XL 0 PASS rc 0; tensor table with the file's tensors +diff UD-Q4_K_XL 0 PASS rc 0 vs embeddinggemma-2-BF16.gguf +hex UD-Q4_K_XL 0 PASS rc 0; GGUF v3 header +tree UD-Q4_K_XL 0 PASS rc 0; root line with the tensor count +flow UD-Q4_K_XL 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding +pull UD-Q5_K_XL 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list UD-Q5_K_XL 0 PASS rc 0; lists the pulled file by its cache path +rm UD-Q5_K_XL 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize UD-Q5_K_XL 0 PASS rc 0; roundtrip: true; no greedy fallback +capability UD-Q5_K_XL 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect UD-Q5_K_XL 0 PASS rc 0; architecture gemma-embedding2; causal false +debug UD-Q5_K_XL 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate UD-Q5_K_XL 0 PASS rc 0; VALID, 0 contract violations +lint UD-Q5_K_XL 0 PASS rc 0; lint passed, 0 errors +tensors UD-Q5_K_XL 0 PASS rc 0; tensor table with the file's tensors +diff UD-Q5_K_XL 0 PASS rc 0 vs embeddinggemma-2-BF16.gguf +hex UD-Q5_K_XL 0 PASS rc 0; GGUF v3 header +tree UD-Q5_K_XL 0 PASS rc 0; root line with the tensor count +flow UD-Q5_K_XL 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding +pull UD-Q6_K_XL 0 PASS rc 0; the cached file's sha256 equals SHA256SUMS +list UD-Q6_K_XL 0 PASS rc 0; lists the pulled file by its cache path +rm UD-Q6_K_XL 0 PASS rc 0; no gguf left; list no longer shows its cache path +tokenize UD-Q6_K_XL 0 PASS rc 0; roundtrip: true; no greedy fallback +capability UD-Q6_K_XL 0 PASS rc 0; the 8 embedding ops listed (file-independent) +inspect UD-Q6_K_XL 0 PASS rc 0; architecture gemma-embedding2; causal false +debug UD-Q6_K_XL 0 PASS rc 0; GGUF gemma-embedding2; health OK +validate UD-Q6_K_XL 0 PASS rc 0; VALID, 0 contract violations +lint UD-Q6_K_XL 0 PASS rc 0; lint passed, 0 errors +tensors UD-Q6_K_XL 0 PASS rc 0; tensor table with the file's tensors +diff UD-Q6_K_XL 0 PASS rc 0 vs embeddinggemma-2-BF16.gguf +hex UD-Q6_K_XL 0 PASS rc 0; GGUF v3 header +tree UD-Q6_K_XL 0 PASS rc 0; root line with the tensor count +flow UD-Q6_K_XL 0 PASS rc 0; embedding path (pool, projection, L2), no LM head; --json kind embedding diff --git a/docs/audits/APR-EMBED-001/EG-1/verb-sweep.md b/docs/audits/APR-EMBED-001/EG-1/verb-sweep.md new file mode 100644 index 0000000000..23db5a0a33 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-1/verb-sweep.md @@ -0,0 +1,105 @@ +# EG-1 verb sweep: the 14 verbs on 6 of 6 files (2026-10-09) + +**Result: 84 / 84 PASS** on the second run (`eg1-sweep2`), 14 verbs × 6 files. The first run scored 75 / 84. Its +9 failures were 6 wrong `list` expectations and 3 `validate` failures from a real defect (BF16 tensors had no +reader). The defect is fixed in 2a42c3d99a and the two expectations are corrected. Nothing else changed between +the runs. + +| Item | Run 1 (`eg1-sweep`) | Run 2 (`eg1-sweep2`) | +|------|--------------------|----------------------| +| apr | release build @ 85ac2d193b (EG-1 slice 5), `apr 0.70.2 (85ac2d193b)` | release build @ 2a42c3d99a (BF16 reader), `apr 0.70.2 (2a42c3d99a)` | +| sha256 | `432fe29ad38db69f400b90cbae7b88df52ca17c495bbf6bc0d180061468998aa` | `50049e707fd8ac5d224e9629b8f5a2187c7d319eca3e8d2f6feaf858c3db3e6d` | +| `cargo test -p apr-cli --lib flow_embedding` | 10 passed, 0 failed | 10 passed, 0 failed | +| Window (UTC) | 2026-10-09 19:58 → 20:34 | 2026-10-09 21:54 → 22:18 | +| Cells | 75 / 84 | 84 / 84 | + +**Files:** `hf://unsloth/embeddinggemma-2-GGUF`: BF16, F16, Q8_0, UD-Q4_K_XL, UD-Q5_K_XL, UD-Q6_K_XL. Each file's +sha256 was checked against a SHA256SUMS before the cells started, and each `pull` cell re-checks the pulled bytes. +All six have 413 tensors. + +**How a cell is scored.** Every verb runs under a timeout with stdin closed. A cell passes only on exit code 0 +**and** a positive check of its output. Neither one alone is enough. `pull`, `list` and `rm` run in a sandboxed +`HOME` and `XDG_CACHE_HOME` per file and fetch the real file from the hub. + +| Verb | Check (beyond rc 0) | +|------|---------------------| +| pull | the cached file's sha256 equals SHA256SUMS | +| list | lists the pulled file by its cache path (run 2; see below) | +| rm | no `.gguf` left in the sandbox, and `list` no longer shows that cache path (run 2; see below) | +| tokenize | `encode -p "hello world"`: `roundtrip: true`, no greedy fallback | +| capability | `--json` lists the 8 embedding ops: BidirectionalMask, SlidingWindowAttn, GatedGeluMlp, PerLayerInput, LayerOutputScale, MeanPool, DenseProjection, L2Normalize | +| inspect | architecture `gemma-embedding2`, `attention.causal` false | +| debug | `GGUF gemma-embedding2`, health OK | +| validate | `VALID: N tensors checked, 0 contract violations` | +| lint | lint passed, 0 errors | +| tensors | tensor table naming `token_embd.weight` or `blk.0.attn_q.weight` | +| diff | `Model Diff` against the BF16 file (F16 for the BF16 row) | +| hex | `Format: GGUF v3` | +| tree | root line ` (N tensors` | +| flow | `(embedding, gemma-embedding2`, `Pool: mean`, `L2 normalise`, `OUTPUT embedding [batch, 768]`; no LM head, logits, next_token or decoder-only text; `--json` has `"kind": "embedding"` | + +## Run 1: 75 / 84 + +| Failed cells | Cause | Class | +|--------------|-------|-------| +| `validate` on BF16 (219 violations), Q8_0 (1), UD-Q6_K_XL (1) | `INVALID: 413 tensors, N contract violations`. Each violation is a BF16 tensor (ggml type 30) that aprender-core could not read: "tensor data could not be read … is BF16 (ggml type 30): aprender-core has no dequantizer". Q8_0 and UD-Q6_K_XL keep one BF16 tensor, `per_layer_model_proj.weight`. UD-Q4_K_XL and UD-Q5_K_XL have none and were VALID | **defect**, fixed | +| `list` on all 6 | The check looked for the file by name. An `hf://` single-file pull writes no manifest (GH-352), so `apr list` shows the file under its cache path, `/pacha/models/.gguf`, as an orphan. The file **was** listed, so the expectation was wrong | **check**, corrected | + +The `rm` cells passed in run 1, but that pass was vacuous. Their "list no longer shows it" half searched for the +file name, which `list` never printed even before `rm`. + +### The fix (2a42c3d99a, Refs #4900) + +BF16 is the top 16 bits of an F32, so it widens to F32 **bit-exactly**: `f32::from_bits(u32::from(bits) << 16)`. +`get_tensor_f32` now reads ggml type 30 next to F16 (type 1), through one bounds-checked two-byte widener. Both +whole-model F32 loads that go through it now accept a BF16 tensor: `get_all_tensors_f32`, which is the GH-375 +fallback of `apr import` and the path `apr convert` uses. Nothing is approximated. That is the difference from the +#3947 IQ decoders, which stay "inspection only". + +New tests in `reader_tests_ggml_sizes.rs`: + +- `test_4900_get_tensor_f32_widens_bf16_bit_exactly`: 8 values, including -0.0, `f32::MIN_POSITIVE`, a + non-trivial mantissa and +∞, are equal `to_bits()`. The fixture first asserts that its low 16 bits are zero, so + the test cannot pass by rounding. +- `test_4900_f16_keeps_its_own_widening`: the F16 arm is unchanged. 0x3C00 is 1.0 in F16 but 2^-7 in BF16, so + sharing the arm cannot silently swap the decoders. +- `test_4900_short_bf16_data_is_refused`: a truncated BF16 tensor is an error naming the tensor ("exceeds file + size"), never a short read. +- `test_4900_whole_model_f32_load_widens_bf16_exactly`: `get_all_tensors_f32` returns the same bits. + +`cargo test -p aprender-core --lib -- test_4900 test_3601 test_3656 test_3947` ran on a tree whose `aprender-core` +is that of 2a42c3d99a: 16 passed, 0 failed, 14294 filtered out. The four `test_4900_*` tests are among the 16. +`cargo clippy -p aprender-core --lib -- -D warnings` is clean on the same tree. Under `--profile test`, clippy +reported one finding in the BF16 test (`verbose_bit_mask`, fixed in de03bbb402: the check is now +`trailing_zeros() >= 16`) and 19 in files this branch does not touch. + +### The corrected checks + +`list` and `rm` now take the cache path from the pull log (` Path: `). `list` must show that path, and after +`rm` the path must be gone. The `rm` check also fails if no path was read, so it can no longer pass vacuously. The +corrected checks were first replayed on run 1's logs: on all 6 files, `list` showed the pulled path and the +`list` taken after `rm` did not. Only the two predicates changed. The binary-independent parts of the script, the +files and the other 12 checks are the same as in run 1. + +## Run 2: 84 / 84 + +All 84 cells passed, 14 on each of the 6 files. The 84 rows (verb, file, exit code, verdict, check) are in +`verb-sweep-cells.tsv`, next to this file. + +- `validate` printed `VALID: 413 tensors checked, 0 contract violations` on all 6 files. That includes BF16, Q8_0 + and UD-Q6_K_XL, the three that failed in run 1, so every BF16 tensor in them was read. +- `list` showed the pulled cache path on all 6 files. After `rm`, that path was gone on all 6. The `rm` check now + needs a path read from the pull log, so these passes are not vacuous. +- The other 11 verbs passed on all 6 files, as they did in run 1. + +## Findings (not fixed here) + +1. **`apr list` after an `hf://` single-file pull shows a hash name.** The pull writes no manifest, so the model + is listed as an orphan `.gguf` under its cache path, never by its repo or file name. This affects every + architecture, not only this one. It needs its own ticket. +2. **`apr pull` suggests `apr run` / `apr serve` for an embedding model.** Its closing hint is the same for every + kind. For `ModelKind::Embedding` the useful next command is `apr embed`. +3. **`apr inspect` prints a bool array as `[len=0]`.** `gemma-embedding2.attention.sliding_window_pattern` is a + 24-entry bool array in all 6 files (5 local, then 1 global, repeated 4 times; read straight from the GGUF header + with a one-off reader). The metadata table shows `[len=0]`, so the per-layer attention type that I-9 reads from + the file cannot be seen with `inspect`. This is a display defect only. It does not change what the file holds. From 6cf18801af043ce6d4509b2cda01946d39eb5fe3 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 16:32:51 +0200 Subject: [PATCH 15/21] feat(embed): EG-2 EmbeddingGemma 2 forward and `apr embed` on GGUF `apr embed` now reads an EmbeddingGemma 2 GGUF file as well as a BERT APR file; the file's magic bytes pick the path. The GGUF path takes its tokenizer, sizes and mean pooling from the file, applies the model card's task prompts (`--task`, `--title`), and keeps a re-normalised Matryoshka prefix with `--dim 512|256|128`. Vectors are unit length. The forward is its own module: centred local window |i - j| <= W/2 on the sliding layers, partial NeoX RoPE with per-kind bases, q/k RMS norms, a weightless v norm, score scale 1.0, a GELU-gated FFN and the per-layer embedding inputs. The embeddinggemma2-forward contract goes to v1.1.0 with the scale and the centred window corrected. Weight shapes are read as [out, in], the order the GGUF reader holds them in, and a fixture test holds the fixture to that order: a fixture written the other way round had let a forward that read the other way round pass every unit test. A BERT APR file without `--vocab` now exits 5 (validation) instead of 2 (usage): the flag is optional because a GGUF file carries its own tokenizer. Parity against the oracle is provisional (the pinned llama.cpp refuses the architecture; moving the pin is D-3): see the EG-2 receipt. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- book/src/cli/embed.md | 2 +- contracts/apr-cli-commands-v1.yaml | 2 +- contracts/embeddinggemma2-forward-v1.yaml | 15 +- crates/apr-cli/src/commands/embed.rs | 30 +- crates/apr-cli/src/commands/embed_gguf.rs | 288 ++++++ .../apr-cli/src/commands/embed_gguf_tests.rs | 284 ++++++ crates/apr-cli/src/commands/mod.rs | 1 + crates/apr-cli/src/dispatch_analysis.rs | 38 +- crates/apr-cli/src/extended_commands.rs | 97 +- .../inference/forward/attention_windowed.rs | 86 ++ .../forward/attention_windowed_tests.rs | 174 ++++ .../forward/embedding_gemma2_input.rs | 185 ++++ .../forward/embedding_gemma2_input_tests.rs | 296 ++++++ .../forward/forward_embedding_gemma2.rs | 755 ++++++++++++++++ .../forward/forward_embedding_gemma2_tests.rs | 596 +++++++++++++ .../src/gguf/inference/forward/mod.rs | 2 + .../src/gguf/inference/fused_matmul_into.rs | 4 +- .../src/gguf/inference/matmul_fused.rs | 24 +- crates/aprender-serve/src/gguf/metadata.rs | 20 +- crates/aprender-serve/src/gguf/mod.rs | 9 + .../aprender-serve/src/gguf/test_factory.rs | 12 + docs/audits/APR-EMBED-001/EG-2/i8-cells.tsv | 841 ++++++++++++++++++ docs/audits/APR-EMBED-001/EG-2/parity.md | 247 +++++ 23 files changed, 3932 insertions(+), 76 deletions(-) create mode 100644 crates/apr-cli/src/commands/embed_gguf.rs create mode 100644 crates/apr-cli/src/commands/embed_gguf_tests.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/attention_windowed.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/attention_windowed_tests.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input_tests.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2.rs create mode 100644 crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2_tests.rs create mode 100644 docs/audits/APR-EMBED-001/EG-2/i8-cells.tsv create mode 100644 docs/audits/APR-EMBED-001/EG-2/parity.md diff --git a/book/src/cli/embed.md b/book/src/cli/embed.md index eae5c956e4..8793db3372 100644 --- a/book/src/cli/embed.md +++ b/book/src/cli/embed.md @@ -2,7 +2,7 @@ # apr embed -Produce sentence embeddings from a BERT bi-encoder (GH-326 Phase 6). +Produce sentence embeddings from a BERT APR file (GH-326 Phase 6) or an EmbeddingGemma 2 GGUF file (APR-EMBED-001). **Category**: Other diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index 79b74a2bba..e577ba2a4d 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -813,7 +813,7 @@ commands: - name: embed category: inference - description: "Produce sentence embeddings from a BERT bi-encoder loaded from an APR file (GH-326 Phase 6). Loads encoder-only `BertModel` checkpoints (e.g. `sentence-transformers/all-MiniLM-L6-v2`), tokenises text with WordPiece, runs the encoder forward, then pools hidden states with `--pool cls` ([CLS] hidden state) or `--pool mean` (mean of token positions; sentence-transformers default). Optionally L2-normalises (`--normalize`, default true). Repeatable `--text` for batch encoding. Outputs `{ text, embedding: [f32; hidden_dim], dim }` per input." + description: "Produce sentence embeddings from a BERT APR file or an EmbeddingGemma 2 GGUF file; the file's magic bytes pick the path. An APR file (GH-326 Phase 6) loads encoder-only `BertModel` checkpoints (e.g. `sentence-transformers/all-MiniLM-L6-v2`), tokenises text with WordPiece (`--vocab`, required), runs the encoder forward, then pools hidden states with `--pool cls` ([CLS] hidden state) or `--pool mean` (mean of token positions; sentence-transformers default), optionally L2-normalised (`--normalize`, default true). A GGUF file (APR-EMBED-001) takes its tokenizer, sizes and mean pooling from the file and gives unit-length vectors; `--task` applies the model card's prompt, `--title` names a `--task document`, and `--dim 512|256|128` keeps a re-normalised Matryoshka prefix. Repeatable `--text` for batch encoding. Outputs `{ text, embedding: [f32; dim], dim }` per input." requires_model: true kinds: {generative: refuse, embedding: receipt} use_instead: {generative: run} diff --git a/contracts/embeddinggemma2-forward-v1.yaml b/contracts/embeddinggemma2-forward-v1.yaml index fd9a228090..0fc6a2bf01 100644 --- a/contracts/embeddinggemma2-forward-v1.yaml +++ b/contracts/embeddinggemma2-forward-v1.yaml @@ -1,5 +1,5 @@ metadata: - version: 1.0.0 + version: 1.1.0 created: '2026-10-07' author: PAIML Engineering description: EmbeddingGemma 2 forward pass -- tokens to a unit-norm, projected, optionally truncated embedding (APR-EMBED-001 EG-0 f) @@ -26,19 +26,20 @@ equations: postconditions: - 'every layer has exactly one kind and the kind set is {local, global}' local_attention: - formula: A_local(x)_i = softmax_j( q_i . k_j / sqrt(d_head) + M_ij ) v_j with M_ij = 0 if |i - j| < W else -inf + formula: A_local(x)_i = softmax_j( q_i . k_j + M_ij ) v_j with M_ij = 0 if |i - j| <= floor(W / 2) else -inf domain: x in R^{n x d_model}, W = sliding_window read from metadata codomain: R^{n x d_model} invariants: - The mask is symmetric (bidirectional); no causal term (attention.causal = false) - - Positions with |i - j| >= W receive exactly zero weight + - Positions with |i - j| > floor(W / 2) receive exactly zero weight; the window is centred, W / 2 on each side (llama.cpp symmetric SWA) + - The score scale is 1.0, not 1 / sqrt(d_head), as in the reference forward preconditions: - W > 0 - n > 0 postconditions: - 'output.shape == input.shape and all output values are finite' global_attention: - formula: A_global(x)_i = softmax_j( q_i . k_j / sqrt(d_head) + P_ij ) v_j with P_ij = 0 for real tokens, -inf for padding + formula: A_global(x)_i = softmax_j( q_i . k_j + P_ij ) v_j with P_ij = 0 for real tokens, -inf for padding domain: x in R^{n x d_model} codomain: R^{n x d_model} invariants: @@ -112,7 +113,7 @@ proof_obligations: - id: EG2-INV-002 type: invariant property: Local attention window - formal: "∀ l ∈ local, ∀ i, j: abs(i - j) ≥ W ⇒ weight(i, j) = 0" + formal: "∀ l ∈ local, ∀ i, j: abs(i - j) > ⌊W/2⌋ ⇒ weight(i, j) = 0 ∧ abs(i - j) ≤ ⌊W/2⌋ ⇒ weight(i, j) > 0" applies_to: all - id: EG2-INV-003 type: invariant @@ -163,8 +164,8 @@ falsification_tests: - id: FALSIFY-EG2-002 rule: Local attention window prediction: Attention weights beyond the window are exactly zero in local layers - test: proptest over n in [1, 3W] with random q, k; inspect the weight matrix - if_fails: Window off by one, or a causal mask applied instead of a symmetric one + test: unit test over n in [1, 3W] with random q, k and a V that marks each position; the boundary pair abs(i - j) = floor(W / 2) is inside, floor(W / 2) + 1 is outside + if_fails: Window off by one, the full W taken on each side instead of W / 2, or a causal mask applied instead of a symmetric one - id: FALSIFY-EG2-003 rule: Bidirectional global attention prediction: Reversing token order permutes, but does not change the set of, attention weights in global layers diff --git a/crates/apr-cli/src/commands/embed.rs b/crates/apr-cli/src/commands/embed.rs index 269f6ae0e2..1036522a3a 100644 --- a/crates/apr-cli/src/commands/embed.rs +++ b/crates/apr-cli/src/commands/embed.rs @@ -209,6 +209,22 @@ fn load_text_file(path: &Path) -> Result> { Ok(out) } +/// The texts to embed: `--text` (in CLI order), then the `--text-file` rows. Phase 7 +/// supports both modes simultaneously so callers can mix a query (`--text "$Q"`) with a +/// passage corpus (`--text-file docs.txt`). The BERT and GGUF paths both read them here. +pub(super) fn collect_texts(texts: &[String], text_file: Option<&Path>) -> Result> { + let mut all_texts: Vec = texts.to_vec(); + if let Some(path) = text_file { + all_texts.extend(load_text_file(path)?); + } + if all_texts.is_empty() { + return Err(CliError::ValidationFailed( + "apr embed requires at least one --text or --text-file row".to_string(), + )); + } + Ok(all_texts) +} + /// Entry point. Loads the encoder once + processes all texts. #[allow(clippy::too_many_arguments)] pub(crate) fn run( @@ -227,21 +243,9 @@ pub(crate) fn run( type_vocab_size: usize, json: bool, ) -> Result<()> { - // Concat `--text` (in CLI order) with `--text-file` rows. Phase 7 - // supports both modes simultaneously so callers can mix a query - // (`--text "$Q"`) with a passage corpus (`--text-file docs.txt`). - let mut all_texts: Vec = texts.to_vec(); - if let Some(path) = text_file { - all_texts.extend(load_text_file(path)?); - } + let all_texts = collect_texts(texts, text_file)?; let texts = all_texts.as_slice(); - if texts.is_empty() { - return Err(CliError::ValidationFailed( - "apr embed requires at least one --text or --text-file row".to_string(), - )); - } - let model_bytes = std::fs::read(model).map_err(|e| { CliError::ValidationFailed(format!("Failed to read {}: {e}", model.display())) })?; diff --git a/crates/apr-cli/src/commands/embed_gguf.rs b/crates/apr-cli/src/commands/embed_gguf.rs new file mode 100644 index 0000000000..56a8f738db --- /dev/null +++ b/crates/apr-cli/src/commands/embed_gguf.rs @@ -0,0 +1,288 @@ +//! `apr embed` on either file kind (APR-EMBED-001 EG-2, #4901). +//! +//! The file's magic bytes pick the path. An APR file takes the BERT path of GH-326 unchanged +//! (`super::embed`). A GGUF file is an EmbeddingGemma 2 model: its tokenizer, sizes and pooling +//! come from the file, so the flags that would restate them are refused rather than ignored, +//! and the vectors come from realizar's forward over the input that `apr serve` builds too +//! (`realizar::gguf::embedding_gemma2_input`). Contract: +//! `contracts/embeddinggemma2-forward-v1.yaml`. + +use super::embed; +use super::model_header::read_prefix; +use crate::error::{CliError, Result}; +use crate::EmbedTask; +use realizar::gguf::embedding_gemma2_input::{EmbedTokenizer, Prompt, Task}; +use realizar::gguf::forward_embedding_gemma2::{truncate_to, EmbeddingGemma2, MATRYOSHKA_DIMS}; +use realizar::gguf::{GGUFModel, MappedGGUFModel}; +use realizar::model_kind::{gguf_model_kind, ModelKind}; +use std::path::Path; + +/// The BERT size overrides; each is `None` unless its flag was given. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct BertSizes { + pub(crate) hidden_dim: Option, + pub(crate) num_layers: Option, + pub(crate) num_heads: Option, + pub(crate) intermediate_dim: Option, + pub(crate) vocab_size: Option, + pub(crate) max_position_embeddings: Option, + pub(crate) type_vocab_size: Option, +} + +impl BertSizes { + /// The long flag of the first size that was given. + fn first_given(&self) -> Option<&'static str> { + [ + ("--hidden-dim", self.hidden_dim), + ("--num-layers", self.num_layers), + ("--num-heads", self.num_heads), + ("--intermediate-dim", self.intermediate_dim), + ("--vocab-size", self.vocab_size), + ("--max-position-embeddings", self.max_position_embeddings), + ("--type-vocab-size", self.type_vocab_size), + ] + .into_iter() + .find_map(|(flag, value)| value.map(|_| flag)) + } +} + +/// The parsed `apr embed` flags. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EmbedFlags<'a> { + pub(crate) texts: &'a [String], + pub(crate) text_file: Option<&'a Path>, + pub(crate) vocab: Option<&'a Path>, + pub(crate) pool: &'a str, + pub(crate) normalize: bool, + pub(crate) task: Option, + pub(crate) title: Option<&'a str>, + pub(crate) dim: Option, + pub(crate) sizes: BertSizes, + pub(crate) json: bool, +} + +/// Entry point: route `model` by its magic bytes. +/// +/// # Errors +/// An unreadable file, a flag the file's kind does not take, and every error of the path taken. +pub(crate) fn run(model: &Path, flags: &EmbedFlags<'_>) -> Result<()> { + let head = read_prefix(model, 4).map_err(|e| { + CliError::ValidationFailed(format!("Failed to read {}: {e}", model.display())) + })?; + if head.as_slice() == b"GGUF" { + run_gguf(model, flags) + } else { + run_bert(model, flags) + } +} + +fn refused(reason: String) -> CliError { + CliError::ValidationFailed(reason) +} + +/// An APR file: the GH-326 BERT path, with its defaults (MiniLM-L6) for the sizes not given. +fn run_bert(model: &Path, flags: &EmbedFlags<'_>) -> Result<()> { + let gguf_only = [ + ("--task", flags.task.is_some()), + ("--title", flags.title.is_some()), + ("--dim", flags.dim.is_some()), + ]; + if let Some((flag, _)) = gguf_only.iter().find(|(_, given)| *given) { + return Err(refused(format!( + "{flag} applies to an EmbeddingGemma 2 GGUF file; {} is not a GGUF file", + model.display() + ))); + } + let vocab = flags.vocab.ok_or_else(|| { + refused(format!( + "--vocab is required for an APR file: {} carries no tokenizer", + model.display() + )) + })?; + let s = flags.sizes; + embed::run( + model, + flags.texts, + flags.text_file, + vocab, + flags.pool, + flags.normalize, + s.hidden_dim.unwrap_or(384), + s.num_layers.unwrap_or(6), + s.num_heads.unwrap_or(12), + s.intermediate_dim.unwrap_or(1536), + s.vocab_size.unwrap_or(30522), + s.max_position_embeddings.unwrap_or(512), + s.type_vocab_size.unwrap_or(2), + flags.json, + ) +} + +/// The flags a GGUF file answers itself, and the combinations that mean nothing, refused before +/// the model is read. +fn refuse_gguf_flags(flags: &EmbedFlags<'_>) -> Result<()> { + if flags.vocab.is_some() { + return Err(refused( + "--vocab is for an APR file; a GGUF file carries its own tokenizer".to_string(), + )); + } + if let Some(flag) = flags.sizes.first_given() { + return Err(refused(format!( + "{flag} is for an APR file; a GGUF file's sizes come from the file" + ))); + } + if flags.pool != "mean" { + return Err(refused(format!( + "--pool {}: an EmbeddingGemma 2 file declares mean pooling, and that is the one it takes", + flags.pool + ))); + } + if !flags.normalize { + return Err(refused( + "--normalize false: an EmbeddingGemma 2 vector is unit length by construction" + .to_string(), + )); + } + if flags.title.is_some() && flags.task != Some(EmbedTask::Document) { + return Err(refused( + "--title names a document; pass it with --task document".to_string(), + )); + } + match flags.dim { + Some(dim) if !MATRYOSHKA_DIMS.contains(&dim) => Err(refused(format!( + "--dim {dim}: the model is trained at {MATRYOSHKA_DIMS:?}" + ))), + _ => Ok(()), + } +} + +/// The prompt `--task` names; no `--task` embeds the text as given. +pub(crate) fn prompt_of(task: Option, title: Option<&str>) -> Prompt<'_> { + let query = match task { + None => return Prompt::Raw, + Some(EmbedTask::Document) => return Prompt::Document { title }, + Some(EmbedTask::SearchQuery) => Task::SearchQuery, + Some(EmbedTask::QuestionAnswering) => Task::QuestionAnswering, + Some(EmbedTask::FactChecking) => Task::FactChecking, + Some(EmbedTask::CodeRetrieval) => Task::CodeRetrieval, + Some(EmbedTask::Classification) => Task::Classification, + Some(EmbedTask::Clustering) => Task::Clustering, + Some(EmbedTask::SentenceSimilarity) => Task::SentenceSimilarity, + }; + Prompt::Query(query) +} + +/// `apr embed` reads an embedding model; a generative GGUF file is refused by kind (I-3). +fn refuse_generative(model: &GGUFModel) -> Result<()> { + match gguf_model_kind(model) { + Ok(ModelKind::Embedding) => Ok(()), + Ok(ModelKind::Generative) => Err(CliError::ModelLoadFailed(format!( + "'{}' is a generative model (ModelKind::Generative): `apr embed` reads an embedding \ + model. Use `apr run`.", + model.architecture().unwrap_or_default() + ))), + Err(reason) => Err(CliError::ModelLoadFailed(reason)), + } +} + +/// One embedded text. +struct Embedded<'a> { + text: &'a str, + tokens: usize, + vector: Vec, +} + +/// A GGUF file: the EmbeddingGemma 2 forward on the file's own tokenizer. +fn run_gguf(model: &Path, flags: &EmbedFlags<'_>) -> Result<()> { + refuse_gguf_flags(flags)?; + let texts = embed::collect_texts(flags.texts, flags.text_file)?; + let load_failed = + |e: realizar::RealizarError| CliError::ModelLoadFailed(format!("{}: {e}", model.display())); + let mapped = MappedGGUFModel::from_path(model).map_err(load_failed)?; + refuse_generative(&mapped.model)?; + let encoder = EmbeddingGemma2::from_mapped(&mapped).map_err(load_failed)?; + let tokenizer = EmbedTokenizer::from_gguf(&mapped.model).map_err(load_failed)?; + let prompt = prompt_of(flags.task, flags.title); + if prompt == Prompt::Raw { + eprintln!( + "note: no --task, so each text is embedded as given; the model card says a task \ + prompt gives better vectors" + ); + } + let mut results = Vec::with_capacity(texts.len()); + for text in &texts { + let tokens = tokenizer.tokens(&prompt, text); + let full = encoder + .embed(&tokens) + .map_err(|e| CliError::InferenceFailed(format!("{text:?}: {e}")))?; + let vector = match flags.dim { + None => full, + Some(dim) => truncate_to(&full, dim).map_err(|e| refused(e.to_string()))?, + }; + results.push(Embedded { + text, + tokens: tokens.len(), + vector, + }); + } + let width = flags.dim.unwrap_or_else(|| encoder.output_dim()); + print_gguf(model, &prompt, width, &results, flags.json); + Ok(()) +} + +fn print_gguf( + model: &Path, + prompt: &Prompt<'_>, + width: usize, + results: &[Embedded<'_>], + json: bool, +) { + if json { + #[allow(clippy::disallowed_methods)] + { + let array: Vec = results + .iter() + .map(|r| { + serde_json::json!({ + "text": r.text, + "embedding": r.vector, + "dim": r.vector.len(), + "tokens": r.tokens, + }) + }) + .collect(); + let payload = serde_json::json!({ + "model": model.display().to_string(), + "kind": "embedding", + "pool": "mean", + "normalize": true, + "prompt": prompt.name(), + "dim": width, + "results": array, + }); + println!( + "{}", + serde_json::to_string_pretty(&payload).unwrap_or_default() + ); + } + return; + } + for r in results { + let preview: Vec = r + .vector + .iter() + .take(4) + .map(|x| format!("{x:+.4}")) + .collect(); + println!( + "text={:?} dim={} preview=[{}, …]", + r.text, + r.vector.len(), + preview.join(", ") + ); + } +} + +#[cfg(test)] +#[path = "embed_gguf_tests.rs"] +mod tests; diff --git a/crates/apr-cli/src/commands/embed_gguf_tests.rs b/crates/apr-cli/src/commands/embed_gguf_tests.rs new file mode 100644 index 0000000000..cda80a05b8 --- /dev/null +++ b/crates/apr-cli/src/commands/embed_gguf_tests.rs @@ -0,0 +1,284 @@ +//! Tests for `embed_gguf` (APR-EMBED-001 EG-2, #4901). +//! +//! Tests cover: +//! - `--task` names the model card's prompts, in the card's order, and `--task document` carries +//! `--title`; the GGUF flags parse with no `--vocab` +//! - A GGUF file refuses each flag it answers itself, and each combination that means nothing, +//! before the model is read; a generative or contradicting file is refused by kind +//! - An APR file refuses the GGUF-only flags and still requires `--vocab` + +use super::*; +use crate::commands::model_kind_gate::tests::eg1_gguf; +use clap::ValueEnum; +use std::io::Write; + +fn texts() -> Vec { + vec!["a".to_string()] +} + +fn flags(texts: &[String]) -> EmbedFlags<'_> { + EmbedFlags { + texts, + text_file: None, + vocab: None, + pool: "mean", + normalize: true, + task: None, + title: None, + dim: None, + sizes: BertSizes::default(), + json: true, + } +} + +fn file(bytes: &[u8]) -> tempfile::NamedTempFile { + let mut f = tempfile::NamedTempFile::new().expect("temp file"); + f.write_all(bytes).expect("write fixture"); + f +} + +fn embedding_file() -> tempfile::NamedTempFile { + file(&eg1_gguf("gemma-embedding2", None)) +} + +fn apr_file() -> tempfile::NamedTempFile { + file(b"APR\0not a GGUF file") +} + +fn assert_refused(model: &Path, flags: &EmbedFlags<'_>, code: u8, needle: &str) { + let err = run(model, flags).expect_err("the call must be refused"); + let msg = err.to_string(); + assert_eq!(err.exit_code_value(), code, "{msg}"); + assert!(msg.contains(needle), "wanted {needle:?} in: {msg}"); +} + +#[test] +fn every_task_names_the_model_card_prompt_eg2() { + let queries: Vec = EmbedTask::value_variants() + .iter() + .filter_map(|t| match prompt_of(Some(*t), None) { + Prompt::Query(task) => Some(task), + _ => None, + }) + .collect(); + assert_eq!(queries, Task::ALL, "the seven query prompts, in order"); + assert_eq!(prompt_of(None, Some("t")), Prompt::Raw); + assert_eq!( + prompt_of(Some(EmbedTask::Document), Some("t")), + Prompt::Document { title: Some("t") } + ); + let names: Vec = EmbedTask::value_variants() + .iter() + .filter_map(|t| t.to_possible_value().map(|v| v.get_name().to_string())) + .collect(); + assert_eq!( + names, + [ + "search-query", + "question-answering", + "fact-checking", + "code-retrieval", + "classification", + "clustering", + "sentence-similarity", + "document", + ] + ); +} + +/// The GGUF flags of a parsed `apr embed` argv: `(vocab given, task, title, dim)`. +type Parsed = (bool, Option, Option, Option); + +/// Parsed on a 16 MiB thread, as the `--normalize` argv tests in `embed.rs` are. +fn parse(argv: &[&str]) -> std::result::Result { + let argv: Vec = argv.iter().map(|s| (*s).to_string()).collect(); + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(move || { + use clap::Parser; + match crate::Cli::try_parse_from(&argv) { + Ok(cli) => match *cli.command { + crate::Commands::Extended(crate::ExtendedCommands::Embed { + vocab, + task, + title, + dim, + .. + }) => Ok(( + vocab.is_some(), + task, + title.map(|t| t.as_str().to_string()), + dim, + )), + other => panic!("expected ExtendedCommands::Embed, got {other:?}"), + }, + Err(e) => Err(e.to_string()), + } + }) + .expect("spawn parse thread") + .join() + .expect("parse thread must not panic") +} + +#[test] +fn the_gguf_flags_parse_without_a_vocab_eg2() { + let parsed = parse(&[ + "apr", "embed", "m.gguf", "--text", "a", "--task", "document", "--title", "T", "--dim", + "256", + ]); + assert_eq!( + parsed, + Ok(( + false, + Some(EmbedTask::Document), + Some("T".to_string()), + Some(256) + )) + ); + let parsed = parse(&["apr", "embed", "m.gguf", "--text", "a"]); + assert_eq!(parsed, Ok((false, None, None, None))); + assert!(parse(&["apr", "embed", "m.gguf", "--task", "summarize"]).is_err()); +} + +#[test] +fn a_gguf_file_refuses_the_flags_it_answers_itself_eg2() { + let model = embedding_file(); + let texts = texts(); + let vocab = Path::new("tokenizer.json"); + let with_vocab = EmbedFlags { + vocab: Some(vocab), + ..flags(&texts) + }; + assert_refused(model.path(), &with_vocab, 5, "--vocab is for an APR file"); + let with_size = EmbedFlags { + sizes: BertSizes { + num_layers: Some(6), + ..BertSizes::default() + }, + ..flags(&texts) + }; + assert_refused( + model.path(), + &with_size, + 5, + "--num-layers is for an APR file", + ); + let cls = EmbedFlags { + pool: "cls", + ..flags(&texts) + }; + assert_refused(model.path(), &cls, 5, "--pool cls"); + let raw = EmbedFlags { + normalize: false, + ..flags(&texts) + }; + assert_refused(model.path(), &raw, 5, "--normalize false"); +} + +#[test] +fn a_gguf_file_refuses_a_title_without_a_document_and_an_untrained_dim_eg2() { + let model = embedding_file(); + let texts = texts(); + for task in [None, Some(EmbedTask::SearchQuery)] { + let titled = EmbedFlags { + task, + title: Some("T"), + ..flags(&texts) + }; + assert_refused(model.path(), &titled, 5, "--title names a document"); + } + for dim in [0, 300, 1024] { + let wide = EmbedFlags { + dim: Some(dim), + ..flags(&texts) + }; + assert_refused(model.path(), &wide, 5, &format!("--dim {dim}")); + } + assert_refused(model.path(), &flags(&[]), 5, "at least one --text"); +} + +#[test] +fn a_gguf_file_that_is_not_an_embedding_model_is_refused_by_kind_eg2() { + let texts = texts(); + let generative = file(&eg1_gguf("llama", None)); + assert_refused(generative.path(), &flags(&texts), 6, "Use `apr run`"); + let causal = file(&eg1_gguf("gemma-embedding2", Some(true))); + assert_refused(causal.path(), &flags(&texts), 6, "attention.causal = true"); +} + +#[test] +fn an_embedding_file_without_the_graph_fails_to_load_eg2() { + // The fixture passes every flag check and the kind gate, then has one tensor of the graph. + let model = embedding_file(); + let texts = texts(); + let document = EmbedFlags { + task: Some(EmbedTask::Document), + title: Some("T"), + dim: Some(256), + ..flags(&texts) + }; + let err = run(model.path(), &document).expect_err("an incomplete graph is refused"); + assert_eq!(err.exit_code_value(), 6, "{err}"); +} + +#[test] +fn an_apr_file_refuses_the_gguf_flags_eg2() { + let model = apr_file(); + let texts = texts(); + let vocab = Path::new("vocab.txt"); + let base = EmbedFlags { + vocab: Some(vocab), + ..flags(&texts) + }; + let cases = [ + ( + EmbedFlags { + task: Some(EmbedTask::SearchQuery), + ..base + }, + "--task", + ), + ( + EmbedFlags { + title: Some("T"), + ..base + }, + "--title", + ), + ( + EmbedFlags { + dim: Some(256), + ..base + }, + "--dim", + ), + ]; + for (case, flag) in &cases { + assert_refused( + model.path(), + case, + 5, + &format!("{flag} applies to an EmbeddingGemma 2 GGUF file"), + ); + } +} + +#[test] +fn an_apr_file_still_requires_a_vocab_eg2() { + let model = apr_file(); + let texts = texts(); + assert_refused( + model.path(), + &flags(&texts), + 5, + "--vocab is required for an APR file", + ); +} + +#[test] +fn an_unreadable_file_is_refused_before_either_path_eg2() { + let dir = tempfile::tempdir().expect("temp dir"); + let missing = dir.path().join("absent.gguf"); + let texts = texts(); + assert_refused(&missing, &flags(&texts), 5, "Failed to read"); +} diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index 357cdc96f6..e584d7580e 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -55,6 +55,7 @@ pub(crate) mod devices; pub(crate) mod dry_sampling_classifier; pub(crate) mod dry_sampling_lint; pub(crate) mod embed; +pub(crate) mod embed_gguf; pub(crate) mod embed_viz; pub(crate) mod embed_viz_classifier; pub(crate) mod embed_viz_lint; diff --git a/crates/apr-cli/src/dispatch_analysis.rs b/crates/apr-cli/src/dispatch_analysis.rs index b87f70c0ad..d06480f7d3 100644 --- a/crates/apr-cli/src/dispatch_analysis.rs +++ b/crates/apr-cli/src/dispatch_analysis.rs @@ -1875,6 +1875,9 @@ fn dispatch_extended_command(cli: &Cli) -> Result<(), CliError> { vocab, pool, normalize, + task, + title, + dim, hidden_dim, num_layers, num_heads, @@ -1883,21 +1886,28 @@ fn dispatch_extended_command(cli: &Cli) -> Result<(), CliError> { max_position_embeddings, type_vocab_size, json, - } => commands::embed::run( + } => commands::embed_gguf::run( model, - &batuta_common::cli_roles::strings(text), - text_file.as_deref(), - vocab, - pool, - *normalize, - *hidden_dim, - *num_layers, - *num_heads, - *intermediate_dim, - *vocab_size, - *max_position_embeddings, - *type_vocab_size, - *json, + &commands::embed_gguf::EmbedFlags { + texts: &batuta_common::cli_roles::strings(text), + text_file: text_file.as_deref(), + vocab: vocab.as_deref(), + pool, + normalize: *normalize, + task: *task, + title: title.as_deref(), + dim: *dim, + sizes: commands::embed_gguf::BertSizes { + hidden_dim: *hidden_dim, + num_layers: *num_layers, + num_heads: *num_heads, + intermediate_dim: *intermediate_dim, + vocab_size: *vocab_size, + max_position_embeddings: *max_position_embeddings, + type_vocab_size: *type_vocab_size, + }, + json: *json, + }, ), // All other extended commands handled by sub-dispatchers above diff --git a/crates/apr-cli/src/extended_commands.rs b/crates/apr-cli/src/extended_commands.rs index da1a52dc90..6c9c63c743 100644 --- a/crates/apr-cli/src/extended_commands.rs +++ b/crates/apr-cli/src/extended_commands.rs @@ -30,6 +30,27 @@ pub enum EmbedProjection { Umap, } +/// The prompt `apr embed --task` puts around each text for an EmbeddingGemma 2 GGUF file. +#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)] +pub enum EmbedTask { + /// `task: search result | query: {text}` + SearchQuery, + /// `task: question answering | query: {text}` + QuestionAnswering, + /// `task: fact checking | query: {text}` + FactChecking, + /// `task: code retrieval | query: {text}` + CodeRetrieval, + /// `task: classification | query: {text}` + Classification, + /// `task: clustering | query: {text}` + Clustering, + /// `task: sentence similarity | query: {text}` + SentenceSimilarity, + /// `title: {--title or none} | text: {text}`: a document to be retrieved. + Document, +} + /// Extended CLI commands (analysis, profiling, QA, benchmarks, and advanced tools). /// /// Flattened into `Commands` via `#[command(flatten)]` so all subcommands remain @@ -1368,9 +1389,12 @@ pub enum ExtendedCommands { #[arg(long)] json: bool, }, - /// Produce sentence embeddings from a BERT bi-encoder (GH-326 Phase 6). + /// Produce sentence embeddings from a BERT APR file or an EmbeddingGemma 2 GGUF file. + /// + /// First-stage dense retrieval companion to `apr rerank`. The file's + /// magic bytes pick the path. /// - /// First-stage dense retrieval companion to `apr rerank`. Loads an + /// An APR file (GH-326 Phase 6) loads an /// encoder-only BertModel (e.g. `sentence-transformers/all-MiniLM-L6-v2`), /// tokenises the input text with WordPiece, runs the full encoder /// forward, then pools the hidden states with one of: @@ -1379,8 +1403,15 @@ pub enum ExtendedCommands { /// sentence-transformers convention) /// Optionally L2-normalises the result (`--normalize`, default true, /// matches sentence-transformers). + /// + /// A GGUF file (APR-EMBED-001) is an EmbeddingGemma 2 model: its + /// tokenizer, sizes and pooling come from the file, and its vectors are + /// unit length. `--task` puts the model card's prompt around each text, + /// `--title` names a `--task document`, and `--dim` keeps the leading + /// 512, 256 or 128 values, re-normalised. Embed { - /// Path to the APR file containing the encoder weights (BertModel). + /// The model: an APR file holding BertModel encoder weights, or an + /// EmbeddingGemma 2 GGUF file. #[arg(value_name = "MODEL")] model: ModelPath, /// Text to encode. Repeatable: `apr embed model.apr --text "a" --text "b" --vocab tok.json`. @@ -1394,9 +1425,11 @@ pub enum ExtendedCommands { /// (50-100 documents) is the embed input. #[arg(long, value_name = "FILE")] text_file: Option, - /// Path to a WordPiece `vocab.txt` or HF `tokenizer.json`. + /// Path to a WordPiece `vocab.txt` or HF `tokenizer.json`. Required + /// for an APR file; refused for a GGUF file, which carries its own + /// tokenizer. #[arg(long, value_name = "FILE")] - vocab: InputFile, + vocab: Option, /// Pooling strategy (`cls` or `mean`). Default: `mean` /// (matches sentence-transformers convention). #[arg(long, default_value = "mean")] @@ -1421,27 +1454,39 @@ pub enum ExtendedCommands { action = clap::ArgAction::Set, )] normalize: bool, - /// Override hidden_dim (default: 384 / MiniLM). - #[arg(long, default_value_t = 384)] - hidden_dim: usize, - /// Override num_layers (default: 6 / MiniLM-L-6). - #[arg(long, default_value_t = 6)] - num_layers: usize, - /// Override num_heads. - #[arg(long, default_value_t = 12)] - num_heads: usize, - /// Override intermediate_dim. - #[arg(long, default_value_t = 1536)] - intermediate_dim: usize, - /// Override vocab_size. - #[arg(long, default_value_t = 30522)] - vocab_size: usize, - /// Override max_position_embeddings. - #[arg(long, default_value_t = 512)] - max_position_embeddings: usize, - /// Override type_vocab_size. - #[arg(long, default_value_t = 2)] - type_vocab_size: usize, + /// GGUF only: the model card's task prompt around each text. Omitted, + /// the text is embedded as given, which the model card says lowers + /// quality. + #[arg(long, value_enum)] + task: Option, + /// GGUF only, with `--task document`: the document's title (default `none`). + #[arg(long, value_name = "TITLE")] + title: Option, + /// GGUF only: keep the leading 512, 256 or 128 values, re-normalised + /// (default: the full 768). + #[arg(long, value_name = "N")] + dim: Option, + /// Override hidden_dim (APR only; default: 384 / MiniLM). + #[arg(long)] + hidden_dim: Option, + /// Override num_layers (APR only; default: 6 / MiniLM-L-6). + #[arg(long)] + num_layers: Option, + /// Override num_heads (APR only; default: 12). + #[arg(long)] + num_heads: Option, + /// Override intermediate_dim (APR only; default: 1536). + #[arg(long)] + intermediate_dim: Option, + /// Override vocab_size (APR only; default: 30522). + #[arg(long)] + vocab_size: Option, + /// Override max_position_embeddings (APR only; default: 512). + #[arg(long)] + max_position_embeddings: Option, + /// Override type_vocab_size (APR only; default: 2). + #[arg(long)] + type_vocab_size: Option, /// Output as JSON. #[arg(long)] json: bool, diff --git a/crates/aprender-serve/src/gguf/inference/forward/attention_windowed.rs b/crates/aprender-serve/src/gguf/inference/forward/attention_windowed.rs new file mode 100644 index 0000000000..320fbe3b9d --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/attention_windowed.rs @@ -0,0 +1,86 @@ +//! APR-EMBED-001 EG-2 (#4901): bidirectional windowed multi-head attention, the shared +//! block every non-causal forward calls (I-10). + +use crate::error::{RealizarError, Result}; +use crate::gguf::OwnedQuantizedModel; + +impl OwnedQuantizedModel { + /// Bidirectional multi-head attention over a whole sequence, with an optional symmetric + /// window: the shared block every non-causal forward calls (APR-EMBED-001 EG-2, I-10). + /// Associated, not a method: it reads no model field. + /// + /// `q` is `[seq_len, num_heads * head_dim]`; `k` and `v` are + /// `[seq_len, num_kv_heads * head_dim]`. Query head `h` reads KV head + /// `h / (num_heads / num_kv_heads)` (grouped-query). Position `i` attends to every `j` + /// with `|i - j| <= half_window`, or to every position when `half_window` is `None`. + /// A position outside the window is never scored, so its weight is exactly 0. There is + /// no causal mask. Scores are `scale * q·k`; the caller passes the scale its model uses. + /// + /// # Errors + /// A slice length that does not match the shape, a zero head count or `head_dim`, or + /// `num_heads` not a multiple of `num_kv_heads`. + #[allow(clippy::too_many_arguments)] + pub(crate) fn parallel_multihead_attention_windowed( + q: &[f32], + k: &[f32], + v: &[f32], + seq_len: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + scale: f32, + half_window: Option, + ) -> Result> { + use rayon::prelude::*; + + let q_width = num_heads * head_dim; + let kv_width = num_kv_heads * head_dim; + if num_heads == 0 || num_kv_heads == 0 || head_dim == 0 || num_heads % num_kv_heads != 0 { + return Err(RealizarError::InvalidShape { + reason: format!( + "windowed attention: {num_heads} query heads over {num_kv_heads} KV heads \ + of width {head_dim}" + ), + }); + } + if q.len() != seq_len * q_width || k.len() != seq_len * kv_width || k.len() != v.len() { + return Err(RealizarError::InvalidShape { + reason: format!( + "windowed attention: q {} / k {} / v {} elements for {seq_len} positions \ + of {q_width} / {kv_width}", + q.len(), + k.len(), + v.len() + ), + }); + } + let group = num_heads / num_kv_heads; + let mut out = vec![0.0f32; seq_len * q_width]; + out.par_chunks_mut(q_width.max(1)) + .enumerate() + .for_each(|(i, row)| { + let (lo, hi) = half_window.map_or((0, seq_len), |w| { + ( + i.saturating_sub(w), + i.saturating_add(w).saturating_add(1).min(seq_len), + ) + }); + let mut weights = vec![0.0f32; hi - lo]; + for (h, out_h) in row.chunks_exact_mut(head_dim).enumerate() { + let kv = (h / group) * head_dim; + let q_h = &q[i * q_width + h * head_dim..][..head_dim]; + for (w, j) in weights.iter_mut().zip(lo..hi) { + *w = scale * crate::apr::simd_dot(q_h, &k[j * kv_width + kv..][..head_dim]); + } + crate::gguf::ops::softmax(&mut weights); + for (w, j) in weights.iter().zip(lo..hi) { + let v_j = &v[j * kv_width + kv..][..head_dim]; + for (o, x) in out_h.iter_mut().zip(v_j) { + *o += w * x; + } + } + } + }); + Ok(out) + } +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/attention_windowed_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/attention_windowed_tests.rs new file mode 100644 index 0000000000..10cdde8a23 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/attention_windowed_tests.rs @@ -0,0 +1,174 @@ +//! Tests for `OwnedQuantizedModel::parallel_multihead_attention_windowed` (attention_windowed.rs), +//! the bidirectional windowed attention block of the embedding forward (APR-EMBED-001 EG-2). +//! +//! Tests cover: +//! - A hand-computed two-position case (the softmax weights and the weighted sum) +//! - The window: a position outside |i - j| <= w has no effect; one inside it does, in +//! both directions (no causal mask) +//! - `None` equals a window at least the sequence length +//! - Grouped-query head mapping +//! - The scale reaches the scores +//! - Shape refusals + +use crate::gguf::OwnedQuantizedModel; + +/// Deterministic values in roughly [-1, 1]: no RNG, so a failure reproduces exactly. +fn ramp(n: usize, seed: f32) -> Vec { + (0..n).map(|i| ((i as f32 + seed) * 0.37).sin()).collect() +} + +fn attend( + q: &[f32], + k: &[f32], + v: &[f32], + seq_len: usize, + heads: (usize, usize), + head_dim: usize, + half_window: Option, +) -> Vec { + OwnedQuantizedModel::parallel_multihead_attention_windowed( + q, + k, + v, + seq_len, + heads.0, + heads.1, + head_dim, + 1.0, + half_window, + ) + .expect("valid shape") +} + +/// One head of width 1: position 0 scores [0, ln 3], so its weights are [1/4, 3/4] and its +/// output is 4/4 + 3·8/4 = 7. +#[test] +fn two_positions_weigh_values_by_the_softmax_of_the_scores_eg2() { + let q = [1.0, 1.0]; + let k = [0.0, 3.0f32.ln()]; + let v = [4.0, 8.0]; + let out = attend(&q, &k, &v, 2, (1, 1), 1, None); + assert!((out[0] - 7.0).abs() < 1e-5, "{out:?}"); + // Position 1 scores the same keys with the same query, so it is the same mixture. + assert!((out[1] - 7.0).abs() < 1e-5, "{out:?}"); +} + +/// Position 0 attends to position 1, which follows it: no causal mask. +#[test] +fn a_later_position_changes_an_earlier_output_eg2() { + let (seq, d) = (4, 4); + let (q, k, mut v) = (ramp(seq * d, 0.0), ramp(seq * d, 1.0), ramp(seq * d, 2.0)); + let before = attend(&q, &k, &v, seq, (1, 1), d, None); + v[3 * d] += 5.0; + let after = attend(&q, &k, &v, seq, (1, 1), d, None); + assert_ne!(before[..d], after[..d], "row 0 ignored a later position"); +} + +/// With w = 1, row i reads rows i-1..=i+1. Changing row 5's key and value moves rows 4 and 5 +/// only. +#[test] +fn a_position_outside_the_window_has_no_effect_eg2() { + let (seq, d) = (6, 4); + let (q, mut k, mut v) = (ramp(seq * d, 0.0), ramp(seq * d, 1.0), ramp(seq * d, 2.0)); + let before = attend(&q, &k, &v, seq, (1, 1), d, Some(1)); + for x in &mut k[5 * d..] { + *x += 3.0; + } + for x in &mut v[5 * d..] { + *x -= 4.0; + } + let after = attend(&q, &k, &v, seq, (1, 1), d, Some(1)); + assert_eq!(before[..4 * d], after[..4 * d], "rows 0..=3 read row 5"); + assert_ne!( + before[4 * d..5 * d], + after[4 * d..5 * d], + "row 4 ignored row 5" + ); + // The edge is inclusive and symmetric: row 0 reads row 1. + let mut v1 = ramp(seq * d, 2.0); + v1[d] += 5.0; + let moved = attend(&q, &ramp(seq * d, 1.0), &v1, seq, (1, 1), d, Some(1)); + assert_ne!( + before[..d], + moved[..d], + "row 0 ignored row 1, inside |i - j| <= 1" + ); +} + +#[test] +fn no_window_equals_a_window_of_at_least_the_sequence_length_eg2() { + let (seq, d) = (5, 8); + let (q, k, v) = ( + ramp(seq * 2 * d, 0.0), + ramp(seq * d, 1.0), + ramp(seq * d, 2.0), + ); + let full = attend(&q, &k, &v, seq, (2, 1), d, None); + assert_eq!(full, attend(&q, &k, &v, seq, (2, 1), d, Some(seq))); + assert_eq!(full, attend(&q, &k, &v, seq, (2, 1), d, Some(usize::MAX))); + assert_ne!(full, attend(&q, &k, &v, seq, (2, 1), d, Some(1))); +} + +/// Four query heads over two KV heads: heads 0 and 1 read KV head 0 (values all 1), heads 2 +/// and 3 read KV head 1 (values all 2). The weights sum to 1, so each output is its value. +#[test] +fn query_heads_read_their_grouped_kv_head_eg2() { + let (seq, d) = (3, 2); + let q = ramp(seq * 4 * d, 0.0); + let k = ramp(seq * 2 * d, 1.0); + let v: Vec = (0..seq).flat_map(|_| [1.0, 1.0, 2.0, 2.0]).collect(); + let out = attend(&q, &k, &v, seq, (4, 2), d, None); + for (i, x) in out.iter().enumerate() { + let head = (i % (4 * d)) / d; + let want = if head < 2 { 1.0 } else { 2.0 }; + assert!((x - want).abs() < 1e-5, "element {i} (head {head}) = {x}"); + } +} + +/// Scale 0 makes every score 0, so each output is the plain mean of the values in its window. +#[test] +fn the_scale_multiplies_the_scores_eg2() { + let (seq, d) = (3, 1); + let (q, k) = (ramp(seq, 0.0), ramp(seq, 1.0)); + let v = [3.0, 6.0, 12.0]; + let out = OwnedQuantizedModel::parallel_multihead_attention_windowed( + &q, &k, &v, seq, 1, 1, d, 0.0, None, + ) + .expect("valid shape"); + for x in &out { + assert!((x - 7.0).abs() < 1e-5, "{out:?}"); + } + assert_ne!( + out, + attend(&q, &k, &v, seq, (1, 1), d, None), + "scale 1 equals scale 0" + ); +} + +#[test] +fn shapes_that_do_not_fit_are_refused_eg2() { + let call = |q: usize, kv: usize, heads: usize, kv_heads: usize, head_dim: usize| { + OwnedQuantizedModel::parallel_multihead_attention_windowed( + &vec![0.0; q], + &vec![0.0; kv], + &vec![0.0; kv], + 2, + heads, + kv_heads, + head_dim, + 1.0, + None, + ) + }; + assert!(call(16, 8, 4, 2, 2).is_ok(), "the control shape is valid"); + assert!(call(16, 12, 4, 3, 2).is_err(), "4 heads over 3 KV heads"); + assert!(call(0, 0, 0, 1, 2).is_err(), "no query heads"); + assert!(call(0, 0, 4, 0, 2).is_err(), "no KV heads"); + assert!(call(0, 0, 4, 2, 0).is_err(), "zero head width"); + assert!(call(15, 8, 4, 2, 2).is_err(), "q one short"); + assert!(call(16, 7, 4, 2, 2).is_err(), "k and v one short"); + let ragged = OwnedQuantizedModel::parallel_multihead_attention_windowed( + &[0.0; 16], &[0.0; 8], &[0.0; 6], 2, 4, 2, 2, 1.0, None, + ); + assert!(ragged.is_err(), "v shorter than k"); +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input.rs b/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input.rs new file mode 100644 index 0000000000..622521a787 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input.rs @@ -0,0 +1,185 @@ +//! APR-EMBED-001 EG-2 (M5): what EmbeddingGemma 2 reads. +//! +//! The model card's seven task prompts, the document form, and the token ids of a prompt: the +//! file's SPM-style BPE vocabulary, with BOS and EOS where the file asks for them +//! (`tokenizer.ggml.add_bos_token`, `tokenizer.ggml.add_eos_token`), as llama.cpp frames an +//! embedding input. `apr embed` and the serve routes (EG-3) both build their input here, so +//! their vectors can differ only in the forward. +//! +//! The tokenizer is fail-closed: a file whose vocabulary is not SPM-style BPE, or that lacks +//! merges, a framing flag or the special id a flag asks for, is refused rather than encoded by +//! the greedy fallback, which is not the model's tokenization. + +use std::sync::Arc; + +use crate::error::{RealizarError, Result}; +use crate::gguf::spm_bpe::SpmBpe; +use crate::gguf::{GGUFModel, GGUFValue}; + +/// The seven query prompts of the model card (M5). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Task { + /// `task: search result | query: …` + SearchQuery, + /// `task: question answering | query: …` + QuestionAnswering, + /// `task: fact checking | query: …` + FactChecking, + /// `task: code retrieval | query: …` + CodeRetrieval, + /// `task: classification | query: …` + Classification, + /// `task: clustering | query: …` + Clustering, + /// `task: sentence similarity | query: …` + SentenceSimilarity, +} + +impl Task { + /// Every task, in the model card's order. + pub const ALL: [Self; 7] = [ + Self::SearchQuery, + Self::QuestionAnswering, + Self::FactChecking, + Self::CodeRetrieval, + Self::Classification, + Self::Clustering, + Self::SentenceSimilarity, + ]; + + /// The words after `task: ` in the query prompt. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::SearchQuery => "search result", + Self::QuestionAnswering => "question answering", + Self::FactChecking => "fact checking", + Self::CodeRetrieval => "code retrieval", + Self::Classification => "classification", + Self::Clustering => "clustering", + Self::SentenceSimilarity => "sentence similarity", + } + } +} + +/// How the model is asked to read a text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Prompt<'a> { + /// The text alone. M5: the model card warns this lowers quality, with no error. + Raw, + /// `task: {label} | query: {text}`. + Query(Task), + /// `title: {title or none} | text: {text}`. + Document { + /// The document's title; `none` when absent. + title: Option<&'a str>, + }, +} + +impl Prompt<'_> { + /// The string the tokenizer reads for `text`. + #[must_use] + pub fn apply(&self, text: &str) -> String { + match self { + Self::Raw => text.to_string(), + Self::Query(task) => format!("task: {} | query: {text}", task.label()), + Self::Document { title } => { + format!("title: {} | text: {text}", title.unwrap_or("none")) + }, + } + } + + /// A short name for receipts (I-12): `raw`, the task label, or `document`. + #[must_use] + pub fn name(&self) -> &'static str { + match self { + Self::Raw => "raw", + Self::Query(task) => task.label(), + Self::Document { .. } => "document", + } + } +} + +fn refuse(reason: impl Into) -> RealizarError { + RealizarError::UnsupportedOperation { + operation: "embedding_gemma2_tokenizer".to_string(), + reason: reason.into(), + } +} + +/// A framing flag the file must declare: `tokenizer.ggml.add_{bos,eos}_token`. +fn declared_flag(model: &GGUFModel, key: &str) -> Result { + match model.metadata.get(key) { + Some(GGUFValue::Bool(flag)) => Ok(*flag), + Some(other) => Err(refuse(format!("{key} is not a bool: {other:?}"))), + None => Err(refuse(format!( + "{key} missing; the framing of an embedding input is not guessed" + ))), + } +} + +/// The special id a framing flag asks for, inside the vocabulary. +fn special_id(id: Option, name: &str, vocab_len: usize) -> Result { + let id = id.ok_or_else(|| { + refuse(format!( + "tokenizer.ggml.add_{name}_token is true but tokenizer.ggml.{name}_token_id is missing" + )) + })?; + if id as usize >= vocab_len { + return Err(refuse(format!( + "tokenizer.ggml.{name}_token_id {id} is outside the vocabulary of {vocab_len}" + ))); + } + Ok(id) +} + +/// The tokenizer of an EmbeddingGemma 2 file, built once and used for every text. +#[derive(Debug, Clone)] +pub struct EmbedTokenizer { + bpe: Arc, + bos: Option, + eos: Option, +} + +impl EmbedTokenizer { + /// Build the tokenizer the file declares. + /// + /// # Errors + /// No vocabulary, a vocabulary that is not SPM-style BPE or has no merges, a missing or + /// non-bool framing flag, or a missing or out-of-range special id that a flag asks for. + pub fn from_gguf(model: &GGUFModel) -> Result { + let vocab = model + .vocabulary() + .ok_or_else(|| refuse("tokenizer.ggml.tokens missing"))?; + let bpe = SpmBpe::from_gguf(&model.metadata, &vocab) + .map_err(|refusal| refuse(refusal.to_string()))?; + let add_bos = declared_flag(model, "tokenizer.ggml.add_bos_token")?; + let add_eos = declared_flag(model, "tokenizer.ggml.add_eos_token")?; + let bos = if add_bos { + Some(special_id(model.bos_token_id(), "bos", vocab.len())?) + } else { + None + }; + let eos = if add_eos { + Some(special_id(model.eos_token_id(), "eos", vocab.len())?) + } else { + None + }; + Ok(Self { bpe, bos, eos }) + } + + /// The framed token ids of `prompt` applied to `text`. + #[must_use] + pub fn tokens(&self, prompt: &Prompt<'_>, text: &str) -> Vec { + let body = self.bpe.encode(&prompt.apply(text)); + let mut ids = Vec::with_capacity(body.len() + 2); + ids.extend(self.bos); + ids.extend(body); + ids.extend(self.eos); + ids + } +} + +#[cfg(test)] +#[path = "embedding_gemma2_input_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input_tests.rs new file mode 100644 index 0000000000..c9c1da3b86 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/embedding_gemma2_input_tests.rs @@ -0,0 +1,296 @@ +//! Tests for `embedding_gemma2_input` (APR-EMBED-001 EG-2, M5). +//! +//! Tests cover: +//! - Every task prompt and the document form equal the corpus prompt table, row for row +//! - The framing: BOS and EOS where the file's flags ask for them, and nowhere else +//! - Special tokens in the text are matched, as llama.cpp does with `parse_special` +//! - Refusals: a vocabulary that is not SPM-style BPE, no merges, a missing or non-bool flag, +//! a missing or out-of-range special id + +use super::{EmbedTokenizer, Prompt, Task}; +use crate::gguf::test_factory::GGUFBuilder; +use crate::gguf::GGUFModel; + +const NORMAL: i32 = 1; +const CONTROL: i32 = 3; + +/// The vocabulary of the fixture: the four specials at the ids the real files use, then pieces. +const TOKENS: [&str; 10] = [ + "", + "", + "", + "", + "a", + "b", + "ab", + "\u{2581}", + "\u{2581}a", + "\u{2581}ab", +]; + +/// Ranked merges: `▁ a` first, so "ab ab" spells `ab` then `▁ab`. +const MERGES: [&str; 3] = ["\u{2581} a", "a b", "\u{2581}a b"]; + +/// The fixture's tokenizer metadata; a field set to `None` (or `false`) leaves its key out. +struct Vocab { + model: &'static str, + merges: bool, + add_bos: Option, + add_eos: Option, + add_eos_as_u32: bool, + bos: Option, + eos: Option, +} + +impl Default for Vocab { + fn default() -> Self { + Self { + model: "gemma4", + merges: true, + add_bos: Some(true), + add_eos: Some(true), + add_eos_as_u32: false, + bos: Some(2), + eos: Some(1), + } + } +} + +impl Vocab { + fn model(&self) -> GGUFModel { + let types: Vec = TOKENS + .iter() + .map(|t| if t.starts_with('<') { CONTROL } else { NORMAL }) + .collect(); + let mut b = GGUFBuilder::new() + .architecture("gemma-embedding2") + .add_string("tokenizer.ggml.model", self.model) + .add_string_array("tokenizer.ggml.tokens", &TOKENS) + .add_i32_array("tokenizer.ggml.token_type", &types); + if self.merges { + b = b.add_string_array("tokenizer.ggml.merges", &MERGES); + } + if let Some(flag) = self.add_bos { + b = b.add_bool("tokenizer.ggml.add_bos_token", flag); + } + if self.add_eos_as_u32 { + b = b.add_u32("tokenizer.ggml.add_eos_token", 1); + } else if let Some(flag) = self.add_eos { + b = b.add_bool("tokenizer.ggml.add_eos_token", flag); + } + if let Some(id) = self.bos { + b = b.add_u32("tokenizer.ggml.bos_token_id", id); + } + if let Some(id) = self.eos { + b = b.add_u32("tokenizer.ggml.eos_token_id", id); + } + GGUFModel::from_bytes(&b.build()).expect("fixture parses") + } + + fn tokenizer(&self) -> EmbedTokenizer { + EmbedTokenizer::from_gguf(&self.model()).expect("fixture tokenizer builds") + } + + fn refused(&self, needle: &str) { + let err = EmbedTokenizer::from_gguf(&self.model()) + .expect_err("the fixture must be refused") + .to_string(); + assert!(err.contains(needle), "wanted {needle:?} in: {err}"); + } +} + +/// `docs/audits/APR-EMBED-001/corpus/prompts.tsv`: `name \t query_template \t document_template`. +fn prompt_table() -> Vec<(String, String, String)> { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../docs/audits/APR-EMBED-001/corpus/prompts.tsv" + ); + let text = std::fs::read_to_string(path).expect("the corpus prompt table is in the workspace"); + text.lines() + .skip(1) + .map(|line| { + let cols: Vec<&str> = line.split('\t').collect(); + assert_eq!(cols.len(), 3, "row {line:?}"); + ( + cols[0].to_string(), + cols[1].to_string(), + cols[2].to_string(), + ) + }) + .collect() +} + +fn task_named(name: &str) -> Task { + let task = match name { + "SearchQuery" => Task::SearchQuery, + "QuestionAnswering" => Task::QuestionAnswering, + "FactChecking" => Task::FactChecking, + "CodeRetrieval" => Task::CodeRetrieval, + "Classification" => Task::Classification, + "Clustering" => Task::Clustering, + "SentenceSimilarity" => Task::SentenceSimilarity, + other => panic!("no task named {other}"), + }; + assert_eq!(format!("{task:?}"), name, "the table names the variant"); + task +} + +#[test] +fn every_prompt_equals_the_corpus_prompt_table_eg2() { + let rows = prompt_table(); + let mut tasks = Vec::new(); + for (name, query, document) in &rows { + if name == "Document" { + assert_eq!(query, "-"); + assert_eq!( + Prompt::Document { title: None }.apply("{content}"), + *document + ); + continue; + } + let task = task_named(name); + tasks.push(task); + assert_eq!(Prompt::Query(task).apply("{content}"), *query, "{name}"); + if document != "-" { + assert_eq!( + Prompt::Document { + title: Some("{title}") + } + .apply("{content}"), + *document, + "{name}" + ); + } + } + assert_eq!(tasks, Task::ALL, "the table lists every task, in order"); + assert_eq!( + rows.len(), + Task::ALL.len() + 1, + "seven tasks plus the document form" + ); +} + +#[test] +fn the_raw_prompt_is_the_text_alone_eg2() { + assert_eq!(Prompt::Raw.apply("a b"), "a b"); + assert_eq!(Prompt::Raw.name(), "raw"); + assert_eq!(Prompt::Query(Task::Clustering).name(), "clustering"); + assert_eq!(Prompt::Document { title: Some("t") }.name(), "document"); +} + +#[test] +fn bos_and_eos_frame_the_body_when_the_file_asks_eg2() { + let tok = Vocab::default().tokenizer(); + // "ab ab" -> "ab▁ab": `▁ a`, then `a b`, then `▁a b`, so two pieces. + assert_eq!(tok.tokens(&Prompt::Raw, "ab ab"), [2, 6, 9, 1]); + assert_eq!( + tok.tokens(&Prompt::Raw, ""), + [2, 1], + "an empty text is the frame alone" + ); +} + +#[test] +fn a_flag_that_is_false_adds_nothing_eg2() { + let bos_only = Vocab { + add_eos: Some(false), + ..Vocab::default() + }; + assert_eq!(bos_only.tokenizer().tokens(&Prompt::Raw, "ab"), [2, 6]); + let eos_only = Vocab { + add_bos: Some(false), + ..Vocab::default() + }; + assert_eq!(eos_only.tokenizer().tokens(&Prompt::Raw, "ab"), [6, 1]); + // An id the file never uses need not exist when its flag is false. + let bare = Vocab { + add_bos: Some(false), + add_eos: Some(false), + bos: None, + eos: Some(99), + ..Vocab::default() + }; + assert_eq!(bare.tokenizer().tokens(&Prompt::Raw, "ab"), [6]); +} + +#[test] +fn special_tokens_in_the_text_are_matched_eg2() { + let tok = Vocab::default().tokenizer(); + assert_eq!(tok.tokens(&Prompt::Raw, "ab"), [2, 6, 1, 1]); +} + +#[test] +fn the_prompt_reaches_the_tokens_eg2() { + let tok = Vocab::default().tokenizer(); + let prompts = [ + Prompt::Query(Task::SearchQuery), + Prompt::Document { title: None }, + Prompt::Document { title: Some("ab") }, + ]; + for prompt in prompts { + assert_eq!( + tok.tokens(&prompt, "ab"), + tok.tokens(&Prompt::Raw, &prompt.apply("ab")), + "{prompt:?}" + ); + } + assert_ne!( + tok.tokens(&Prompt::Raw, "ab"), + tok.tokens(&prompts[0], "ab") + ); +} + +#[test] +fn a_vocabulary_that_is_not_spm_bpe_is_refused_eg2() { + Vocab { + model: "llama", + ..Vocab::default() + } + .refused("tokenizer.ggml.model 'llama' is not SPM-style BPE"); +} + +#[test] +fn a_vocabulary_without_merges_is_refused_eg2() { + Vocab { + merges: false, + ..Vocab::default() + } + .refused("no tokenizer.ggml.merges"); +} + +#[test] +fn a_missing_framing_flag_is_refused_eg2() { + Vocab { + add_bos: None, + ..Vocab::default() + } + .refused("tokenizer.ggml.add_bos_token missing"); + Vocab { + add_eos: None, + ..Vocab::default() + } + .refused("tokenizer.ggml.add_eos_token missing"); +} + +#[test] +fn a_framing_flag_that_is_not_a_bool_is_refused_eg2() { + Vocab { + add_eos_as_u32: true, + ..Vocab::default() + } + .refused("tokenizer.ggml.add_eos_token is not a bool"); +} + +#[test] +fn a_special_id_the_flag_asks_for_must_exist_eg2() { + Vocab { + bos: None, + ..Vocab::default() + } + .refused("tokenizer.ggml.bos_token_id is missing"); + Vocab { + eos: Some(10), + ..Vocab::default() + } + .refused("tokenizer.ggml.eos_token_id 10 is outside the vocabulary of 10"); +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2.rs new file mode 100644 index 0000000000..fb6a83b289 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2.rs @@ -0,0 +1,755 @@ +//! APR-EMBED-001 EG-2 (#4901): the EmbeddingGemma 2 CPU forward. +//! +//! One unit-length vector per input, following llama.cpp's `gemma-embedding2` graph (b11490): +//! +//! 1. Token rows scaled by `sqrt(hidden)`. +//! 2. Per-layer inputs: `per_layer_model_proj · x / sqrt(hidden)`, RMS-normed in chunks of +//! `per_layer` with `per_layer_proj_norm`. Layer `l` of token `t` is chunk `l` of row `t`. +//! 3. Each layer: RMSNorm → Q/K/V; per-head RMSNorm of Q and K (weighted) and V (unweighted); +//! NEOX RoPE with the layer's `n_rot` and base; bidirectional attention with scale 1.0, over +//! `|i - j| <= sliding_window / 2` on a local layer and every position on a global one; +//! output projection; post-attention norm plus residual; RMSNorm → `down(gelu(gate·x) ⊙ up·x)` +//! → post-FFN norm plus residual; the per-layer branch +//! `post_norm(proj(gelu(inp_gate·h) ⊙ input_l))` added; the sum times `layer_output_scale`. +//! 4. `output_norm`, the `output` projection per token, the mean over tokens, unit L2 length. +//! +//! Every block it runs is a shared one (I-10): `matmul_any` for each projection (the one quant +//! dispatcher), `dequantize_at` for a token row, `parallel_multihead_attention_windowed`, +//! `ops::rms_norm`, `ops::gelu` and `apply_partial_neox_rope`. This file reads a tensor's ggml type +//! only to size its rows; it never branches on it. +//! +//! A file that is not exactly this graph is refused: another architecture, a pooling type other +//! than mean, a causal flag, RoPE scaling, shared KV layers, a sliding-window pattern that +//! disagrees with the query widths, a tensor with other dims, or a tensor count other than the +//! 5 + 17 per layer the graph consumes (llama.cpp refuses the same file). + +use crate::error::{RealizarError, Result}; +use crate::gguf::forward_qwen35::apply_partial_neox_rope; +use crate::gguf::ggml_type_table; +use crate::gguf::keys::{self, arch_key}; +use crate::gguf::ops::{gelu, rms_norm}; +use crate::gguf::{ + GGUFModel, GGUFValue, MappedGGUFModel, OwnedQuantizedModel, OwnedQuantizedTensor, +}; +use crate::model_kind::{self, LayerAttention}; + +/// The Matryoshka widths a pooled vector may be truncated to. +pub const MATRYOSHKA_DIMS: [usize; 4] = [768, 512, 256, 128]; + +/// Tensors outside the blocks: `token_embd`, `per_layer_model_proj`, `per_layer_proj_norm`, +/// `output_norm`, `output`. +const NON_BLOCK_TENSORS: usize = 5; + +/// Tensors in each block. +const BLOCK_TENSORS: usize = 17; + +/// The pooling type llama.cpp writes for mean pooling. +const POOLING_MEAN: usize = 1; + +/// The period of llama.cpp's default sliding-window pattern when the file carries none +/// (`load_swa_pattern(ml, 6)`): layer `i` is local unless `i % 6 == 5`. +const DEFAULT_SWA_PERIOD: usize = 6; + +fn refuse(reason: impl Into) -> RealizarError { + RealizarError::UnsupportedOperation { + operation: "embedding_gemma2".to_string(), + reason: reason.into(), + } +} + +fn meta_usize(model: &GGUFModel, arch: &str, suffix: &str) -> Result { + let key = arch_key(arch, suffix); + let value = model + .metadata + .get(&key) + .ok_or_else(|| refuse(format!("{key} missing")))?; + let wide = match value { + GGUFValue::UInt8(v) => Some(u64::from(*v)), + GGUFValue::UInt16(v) => Some(u64::from(*v)), + GGUFValue::UInt32(v) => Some(u64::from(*v)), + GGUFValue::UInt64(v) => Some(*v), + GGUFValue::Int32(v) => u64::try_from(*v).ok(), + GGUFValue::Int64(v) => u64::try_from(*v).ok(), + _ => None, + }; + wide.and_then(|v| usize::try_from(v).ok()) + .ok_or_else(|| refuse(format!("{key} is not a non-negative integer: {value:?}"))) +} + +fn meta_f32(model: &GGUFModel, arch: &str, suffix: &str) -> Result { + let key = arch_key(arch, suffix); + match model.metadata.get(&key) { + Some(GGUFValue::Float32(v)) if v.is_finite() => Ok(*v), + Some(GGUFValue::Float64(v)) if v.is_finite() => Ok(*v as f32), + Some(other) => Err(refuse(format!("{key} is not a finite float: {other:?}"))), + None => Err(refuse(format!("{key} missing"))), + } +} + +/// The model-wide hyperparameters, read and checked once. +#[derive(Debug, Clone, Copy)] +struct Hparams { + hidden: usize, + per_layer: usize, + layers: usize, + heads: usize, + ff: usize, + out: usize, + eps: f32, + half_window: usize, + context: Option, +} + +impl Hparams { + fn read(model: &GGUFModel, arch: &str) -> Result { + let pooling = meta_usize(model, arch, "pooling_type")?; + if pooling != POOLING_MEAN { + return Err(refuse(format!( + "pooling_type {pooling}; this forward pools by mean (1)" + ))); + } + if let Some(GGUFValue::Bool(true)) = model.metadata.get(&arch_key(arch, "attention.causal")) + { + return Err(refuse( + "attention.causal = true on an embedding architecture", + )); + } + Self::refuse_unsupported_keys(model, arch)?; + let window = meta_usize(model, arch, "attention.sliding_window")?; + let hp = Self { + hidden: meta_usize(model, arch, keys::EMBEDDING_LENGTH)?, + per_layer: meta_usize(model, arch, "embedding_length_per_layer_input")?, + layers: meta_usize(model, arch, keys::BLOCK_COUNT)?, + heads: meta_usize(model, arch, keys::ATTENTION_HEAD_COUNT)?, + ff: meta_usize(model, arch, keys::FEED_FORWARD_LENGTH)?, + out: meta_usize(model, arch, "embedding_length_out")?, + eps: meta_f32(model, arch, keys::ATTENTION_LAYER_NORM_RMS_EPSILON)?, + half_window: window / 2, + context: meta_usize(model, arch, keys::CONTEXT_LENGTH).ok(), + }; + let widths = [ + hp.hidden, + hp.per_layer, + hp.layers, + hp.heads, + hp.ff, + hp.out, + window, + ]; + if widths.contains(&0) { + return Err(refuse(format!( + "a zero width among hidden/per_layer/layers/heads/ff/out/window: {widths:?}" + ))); + } + Ok(hp) + } + + /// RoPE scaling and shared KV layers change the graph; this forward runs neither. + fn refuse_unsupported_keys(model: &GGUFModel, arch: &str) -> Result<()> { + let scaling = arch_key(arch, "rope.scaling."); + if let Some(key) = model.metadata.keys().find(|k| k.starts_with(&scaling)) { + return Err(refuse(format!( + "{key} present; this forward runs RoPE unscaled" + ))); + } + let shared = arch_key(arch, "attention.shared_kv_layers"); + if model.metadata.contains_key(&shared) + && meta_usize(model, arch, "attention.shared_kv_layers")? != 0 + { + return Err(refuse(format!( + "{shared} is not 0; this forward shares no KV" + ))); + } + Ok(()) + } +} + +/// The head width, rotated width and RoPE base of one attention kind. +#[derive(Debug, Clone, Copy, PartialEq)] +struct AttentionShape { + head_dim: usize, + n_rot: usize, + rope_base: f32, +} + +impl AttentionShape { + /// `tail` is `""` for the global layers and `"_swa"` for the local ones. + fn read(model: &GGUFModel, arch: &str, tail: &str) -> Result { + let head_dim = meta_usize(model, arch, &format!("attention.key_length{tail}"))?; + let value_dim = meta_usize(model, arch, &format!("attention.value_length{tail}"))?; + if value_dim != head_dim { + return Err(refuse(format!( + "attention.value_length{tail} {value_dim} differs from key_length{tail} {head_dim}" + ))); + } + let n_rot = meta_usize(model, arch, &format!("rope.dimension_count{tail}"))?; + if n_rot == 0 || n_rot % 2 != 0 || n_rot > head_dim { + return Err(refuse(format!( + "rope.dimension_count{tail} {n_rot} is not an even width in 1..={head_dim}" + ))); + } + let rope_base = meta_f32(model, arch, &format!("rope.freq_base{tail}"))?; + Ok(Self { + head_dim, + n_rot, + rope_base, + }) + } +} + +fn tensor_dims(model: &GGUFModel, name: &str) -> Result> { + let info = model + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| refuse(format!("tensor {name} missing")))?; + info.dims + .iter() + .map(|&d| { + usize::try_from(d).map_err(|_| refuse(format!("{name}: dim {d} overflows usize"))) + }) + .collect() +} + +/// The weight `name`, `out_dim` rows of `in_dim`, its bytes copied out of `data`. The reader +/// reverses GGUF's `ne0, ne1`, so it holds the shape as `[out_dim, in_dim]`, as everywhere else +/// in this crate. Rows must be whole blocks of the tensor's type. +fn weight( + model: &GGUFModel, + data: &[u8], + name: &str, + in_dim: usize, + out_dim: usize, +) -> Result { + let dims = tensor_dims(model, name)?; + if dims != [out_dim, in_dim] { + return Err(refuse(format!( + "{name} has dims {dims:?}, expected [{out_dim}, {in_dim}]" + ))); + } + let info = model + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| refuse(format!("tensor {name} missing")))?; + let traits = ggml_type_table::traits(info.qtype).ok_or_else(|| { + refuse(format!( + "{name}: ggml type {} has no block layout", + info.qtype + )) + })?; + if traits.blck_size == 0 || in_dim % traits.blck_size != 0 { + return Err(refuse(format!( + "{name}: a row of {in_dim} is not whole {} blocks of {}", + traits.name, traits.blck_size + ))); + } + let bytes = (in_dim / traits.blck_size) + .checked_mul(traits.type_size) + .and_then(|row| row.checked_mul(out_dim)) + .ok_or_else(|| refuse(format!("{name}: byte size overflows usize")))?; + let start = usize::try_from(info.offset) + .ok() + .and_then(|offset| model.tensor_data_start.checked_add(offset)) + .ok_or_else(|| refuse(format!("{name}: offset {} overflows usize", info.offset)))?; + let slice = start + .checked_add(bytes) + .and_then(|end| data.get(start..end)) + .ok_or_else(|| { + refuse(format!( + "{name}: {bytes} bytes at {start} run past the file" + )) + })?; + Ok(OwnedQuantizedTensor { + data: slice.to_vec(), + in_dim, + out_dim, + qtype: info.qtype, + }) +} + +/// The one-dimensional tensor `name` of `len` elements, as f32. +fn vector(model: &GGUFModel, data: &[u8], name: &str, len: usize) -> Result> { + let dims = tensor_dims(model, name)?; + if dims != [len] { + return Err(refuse(format!( + "{name} has dims {dims:?}, expected [{len}]" + ))); + } + let values = model.get_tensor_f32(name, data)?; + if values.len() != len { + return Err(refuse(format!( + "{name} read {} values, expected {len}", + values.len() + ))); + } + Ok(values) +} + +/// Row `row` of `table`, dequantized: `in_dim` values from its own bytes. +fn table_row(table: &OwnedQuantizedTensor, row: usize) -> Result> { + if row >= table.out_dim { + return Err(refuse(format!( + "token {row} is outside the vocabulary of {}", + table.out_dim + ))); + } + let row_bytes = table.data.len() / table.out_dim; + GGUFModel::dequantize_at(table.qtype, &table.data, row * row_bytes, table.in_dim) +} + +/// Each layer is local unless the file's pattern (or llama.cpp's default period of 6) says +/// global; the query widths must say the same. +fn check_swa_pattern(model: &GGUFModel, arch: &str, kinds: &[LayerAttention]) -> Result<()> { + let key = arch_key(arch, "attention.sliding_window_pattern"); + let declared: Vec = match model.metadata.get(&key) { + None => (0..kinds.len()) + .map(|i| i % DEFAULT_SWA_PERIOD != DEFAULT_SWA_PERIOD - 1) + .collect(), + Some(GGUFValue::Array(items)) => items + .iter() + .map(|item| match item { + GGUFValue::Bool(local) => Ok(*local), + other => Err(refuse(format!("{key} holds a non-bool: {other:?}"))), + }) + .collect::>()?, + Some(other) => return Err(refuse(format!("{key} is not an array: {other:?}"))), + }; + if declared.len() != kinds.len() { + return Err(refuse(format!( + "{key} has {} entries for {} layers", + declared.len(), + kinds.len() + ))); + } + for (i, (&local, kind)) in declared.iter().zip(kinds).enumerate() { + if local != (*kind == LayerAttention::Local) { + return Err(refuse(format!( + "layer {i}: the sliding-window pattern says local = {local}, the attn_q width says {kind:?}" + ))); + } + } + Ok(()) +} + +#[derive(Debug, Clone)] +struct AttnWeights { + norm: Vec, + q: OwnedQuantizedTensor, + k: OwnedQuantizedTensor, + v: OwnedQuantizedTensor, + o: OwnedQuantizedTensor, + q_norm: Vec, + k_norm: Vec, + post_norm: Vec, +} + +#[derive(Debug, Clone)] +struct FfnWeights { + norm: Vec, + gate: OwnedQuantizedTensor, + up: OwnedQuantizedTensor, + down: OwnedQuantizedTensor, + post_norm: Vec, +} + +#[derive(Debug, Clone)] +struct PerLayerWeights { + inp_gate: OwnedQuantizedTensor, + proj: OwnedQuantizedTensor, + post_norm: Vec, + out_scale: f32, +} + +#[derive(Debug, Clone)] +struct Layer { + heads: usize, + kv_heads: usize, + shape: AttentionShape, + half_window: Option, + attn: AttnWeights, + ffn: FfnWeights, + per_layer: PerLayerWeights, +} + +/// The tensors and hyperparameters a layer is loaded from. +struct Source<'a> { + model: &'a GGUFModel, + data: &'a [u8], + hp: Hparams, +} + +impl Source<'_> { + fn weight( + &self, + layer: usize, + tensor: &str, + in_dim: usize, + out_dim: usize, + ) -> Result { + weight( + self.model, + self.data, + &format!("blk.{layer}.{tensor}.weight"), + in_dim, + out_dim, + ) + } + + fn vector(&self, layer: usize, tensor: &str, len: usize) -> Result> { + vector( + self.model, + self.data, + &format!("blk.{layer}.{tensor}.weight"), + len, + ) + } + + fn attn(&self, i: usize, d: usize, kv_heads: usize) -> Result { + let (h, q_width, kv_width) = (self.hp.hidden, self.hp.heads * d, kv_heads * d); + Ok(AttnWeights { + norm: self.vector(i, "attn_norm", h)?, + q: self.weight(i, "attn_q", h, q_width)?, + k: self.weight(i, "attn_k", h, kv_width)?, + v: self.weight(i, "attn_v", h, kv_width)?, + o: self.weight(i, "attn_output", q_width, h)?, + q_norm: self.vector(i, "attn_q_norm", d)?, + k_norm: self.vector(i, "attn_k_norm", d)?, + post_norm: self.vector(i, "post_attention_norm", h)?, + }) + } + + fn ffn(&self, i: usize) -> Result { + let (h, ff) = (self.hp.hidden, self.hp.ff); + Ok(FfnWeights { + norm: self.vector(i, "ffn_norm", h)?, + gate: self.weight(i, "ffn_gate", h, ff)?, + up: self.weight(i, "ffn_up", h, ff)?, + down: self.weight(i, "ffn_down", ff, h)?, + post_norm: self.vector(i, "post_ffw_norm", h)?, + }) + } + + fn per_layer(&self, i: usize) -> Result { + let (h, p) = (self.hp.hidden, self.hp.per_layer); + Ok(PerLayerWeights { + inp_gate: self.weight(i, "inp_gate", h, p)?, + proj: self.weight(i, "proj", p, h)?, + post_norm: self.vector(i, "post_norm", h)?, + out_scale: self.vector(i, "layer_output_scale", 1)?[0], + }) + } + + fn layer( + &self, + i: usize, + kind: LayerAttention, + shape: AttentionShape, + kv_heads: usize, + ) -> Result { + if kv_heads == 0 || self.hp.heads % kv_heads != 0 { + return Err(refuse(format!( + "layer {i}: {} query heads over {kv_heads} KV heads", + self.hp.heads + ))); + } + let half_window = match kind { + LayerAttention::Local => Some(self.hp.half_window), + LayerAttention::Global => None, + }; + Ok(Layer { + heads: self.hp.heads, + kv_heads, + shape, + half_window, + attn: self.attn(i, shape.head_dim, kv_heads)?, + ffn: self.ffn(i)?, + per_layer: self.per_layer(i)?, + }) + } +} + +/// An EmbeddingGemma 2 model, loaded for the CPU forward. +#[derive(Debug, Clone)] +pub struct EmbeddingGemma2 { + hp: Hparams, + token_embd: OwnedQuantizedTensor, + per_layer_model_proj: OwnedQuantizedTensor, + per_layer_proj_norm: Vec, + layers: Vec, + output_norm: Vec, + output: OwnedQuantizedTensor, +} + +impl EmbeddingGemma2 { + /// Load from a mapped GGUF file. + /// + /// # Errors + /// See [`Self::from_gguf`]. + pub fn from_mapped(mapped: &MappedGGUFModel) -> Result { + Self::from_gguf(&mapped.model, mapped.data()) + } + + /// Load from a parsed GGUF file and its bytes. + /// + /// # Errors + /// Any file that is not exactly the `gemma-embedding2` graph (see the module docs). + pub fn from_gguf(model: &GGUFModel, data: &[u8]) -> Result { + let arch = model + .architecture() + .ok_or_else(|| refuse("general.architecture missing"))?; + if !model_kind::is_embedding_gemma2(arch) { + return Err(refuse(format!( + "architecture '{arch}' is not {}", + model_kind::EMBEDDING_GEMMA2_ARCH + ))); + } + let hp = Hparams::read(model, arch)?; + let expected = NON_BLOCK_TENSORS + BLOCK_TENSORS * hp.layers; + if model.tensors.len() != expected { + return Err(refuse(format!( + "{} tensors; the graph consumes {expected} ({NON_BLOCK_TENSORS} + {BLOCK_TENSORS} x {} layers)", + model.tensors.len(), + hp.layers + ))); + } + let kv_heads = model_kind::num_kv_heads_per_layer(model) + .ok_or_else(|| refuse("attention.head_count_kv missing"))? + .map_err(refuse)?; + let q_widths = (0..hp.layers) + .map(|i| { + tensor_dims(model, &format!("blk.{i}.attn_q.weight")) + .map(|d| d.first().copied().unwrap_or(0)) + }) + .collect::>>()?; + let kinds = model_kind::layer_attention_kinds(&q_widths).map_err(refuse)?; + check_swa_pattern(model, arch, &kinds)?; + let global = AttentionShape::read(model, arch, "")?; + let local = AttentionShape::read(model, arch, "_swa")?; + let src = Source { model, data, hp }; + let layers = kinds + .iter() + .zip(&kv_heads) + .enumerate() + .map(|(i, (&kind, &kv))| { + let shape = if kind == LayerAttention::Global { + global + } else { + local + }; + src.layer(i, kind, shape, kv) + }) + .collect::>>()?; + let vocab = tensor_dims(model, "token_embd.weight")? + .first() + .copied() + .unwrap_or(0); + Ok(Self { + hp, + token_embd: weight(model, data, "token_embd.weight", hp.hidden, vocab)?, + per_layer_model_proj: weight( + model, + data, + "per_layer_model_proj.weight", + hp.hidden, + hp.per_layer * hp.layers, + )?, + per_layer_proj_norm: vector(model, data, "per_layer_proj_norm.weight", hp.per_layer)?, + layers, + output_norm: vector(model, data, "output_norm.weight", hp.hidden)?, + output: weight(model, data, "output.weight", hp.hidden, hp.out)?, + }) + } + + /// The width of a pooled vector before truncation. + #[must_use] + pub fn output_dim(&self) -> usize { + self.hp.out + } + + /// The number of token rows. + #[must_use] + pub fn vocab_size(&self) -> usize { + self.token_embd.out_dim + } + + /// The unit-length mean-pooled embedding of `tokens`, `output_dim()` wide. + /// + /// # Errors + /// No tokens, more than the context length, a token outside the vocabulary, a block error, + /// or a pooled vector of zero or non-finite length. + pub fn embed(&self, tokens: &[u32]) -> Result> { + let seq = tokens.len(); + if seq == 0 { + return Err(refuse("no tokens to embed")); + } + if let Some(maximum) = self.hp.context.filter(|&m| seq > m) { + return Err(RealizarError::ContextLimitExceeded { + provided: seq, + maximum, + }); + } + let scale = (self.hp.hidden as f32).sqrt(); + let mut x = Vec::with_capacity(seq * self.hp.hidden); + for &token in tokens { + let row = table_row(&self.token_embd, token as usize)?; + x.extend(row.iter().map(|v| v * scale)); + } + let inputs = self.per_layer_inputs(&x)?; + for (l, layer) in self.layers.iter().enumerate() { + x = self.layer_forward(l, layer, &x, &inputs, seq)?; + } + let normed = rms_norm(&x, &self.output_norm, self.hp.eps); + let projected = OwnedQuantizedModel::matmul_any(&normed, &self.output)?; + unit_length(mean_rows(&projected, self.hp.out)) + } + + /// `[seq, layers * per_layer]`: chunk `l` of row `t` is layer `l`'s input for token `t`. + fn per_layer_inputs(&self, x: &[f32]) -> Result> { + let mut proj = OwnedQuantizedModel::matmul_any(x, &self.per_layer_model_proj)?; + let inv = 1.0 / (self.hp.hidden as f32).sqrt(); + for v in &mut proj { + *v *= inv; + } + Ok(rms_norm(&proj, &self.per_layer_proj_norm, self.hp.eps)) + } + + fn layer_forward( + &self, + l: usize, + layer: &Layer, + x: &[f32], + inputs: &[f32], + seq: usize, + ) -> Result> { + let eps = self.hp.eps; + let attn = self.attention(layer, &rms_norm(x, &layer.attn.norm, eps), seq)?; + let attn_out = add(&rms_norm(&attn, &layer.attn.post_norm, eps), x); + let ffn = gelu_ffn(&rms_norm(&attn_out, &layer.ffn.norm, eps), &layer.ffn)?; + let pe_in = add(&rms_norm(&ffn, &layer.ffn.post_norm, eps), &attn_out); + + let (p, stride) = (self.hp.per_layer, self.hp.per_layer * self.hp.layers); + let mut gated = OwnedQuantizedModel::matmul_any(&pe_in, &layer.per_layer.inp_gate)?; + gelu(&mut gated); + for (t, row) in gated.chunks_exact_mut(p).enumerate() { + let input = &inputs[t * stride + l * p..][..p]; + row.iter_mut().zip(input).for_each(|(g, i)| *g *= i); + } + let projected = OwnedQuantizedModel::matmul_any(&gated, &layer.per_layer.proj)?; + let pe = rms_norm(&projected, &layer.per_layer.post_norm, eps); + let scale = layer.per_layer.out_scale; + Ok(pe_in + .iter() + .zip(&pe) + .map(|(a, b)| (a + b) * scale) + .collect()) + } + + fn attention(&self, layer: &Layer, h: &[f32], seq: usize) -> Result> { + let eps = self.hp.eps; + let shape = layer.shape; + let d = shape.head_dim; + let mut q = rms_norm( + &OwnedQuantizedModel::matmul_any(h, &layer.attn.q)?, + &layer.attn.q_norm, + eps, + ); + let mut k = rms_norm( + &OwnedQuantizedModel::matmul_any(h, &layer.attn.k)?, + &layer.attn.k_norm, + eps, + ); + let v = rms_norm( + &OwnedQuantizedModel::matmul_any(h, &layer.attn.v)?, + &vec![1.0; d], + eps, + ); + for (pos, row) in q.chunks_exact_mut(layer.heads * d).enumerate() { + apply_partial_neox_rope(row, layer.heads, d, shape.n_rot, pos, shape.rope_base); + } + for (pos, row) in k.chunks_exact_mut(layer.kv_heads * d).enumerate() { + apply_partial_neox_rope(row, layer.kv_heads, d, shape.n_rot, pos, shape.rope_base); + } + let ctx = OwnedQuantizedModel::parallel_multihead_attention_windowed( + &q, + &k, + &v, + seq, + layer.heads, + layer.kv_heads, + d, + 1.0, + layer.half_window, + )?; + OwnedQuantizedModel::matmul_any(&ctx, &layer.attn.o) + } +} + +/// `down(gelu(gate·x) ⊙ up·x)`: llama.cpp's `LLM_FFN_GELU` + `LLM_FFN_PAR`. +fn gelu_ffn(x: &[f32], ffn: &FfnWeights) -> Result> { + let mut gate = OwnedQuantizedModel::matmul_any(x, &ffn.gate)?; + let up = OwnedQuantizedModel::matmul_any(x, &ffn.up)?; + gelu(&mut gate); + gate.iter_mut().zip(&up).for_each(|(g, u)| *g *= u); + OwnedQuantizedModel::matmul_any(&gate, &ffn.down) +} + +fn add(a: &[f32], b: &[f32]) -> Vec { + a.iter().zip(b).map(|(x, y)| x + y).collect() +} + +fn mean_rows(x: &[f32], width: usize) -> Vec { + let mut mean = vec![0.0f32; width]; + let mut rows = 0usize; + for row in x.chunks_exact(width) { + mean.iter_mut().zip(row).for_each(|(m, v)| *m += v); + rows += 1; + } + let inv = 1.0 / rows.max(1) as f32; + for m in &mut mean { + *m *= inv; + } + mean +} + +/// `v / |v|`, the norm summed in f64 as llama.cpp's `common_embd_normalize` does. +fn unit_length(mut v: Vec) -> Result> { + let norm = v + .iter() + .map(|&x| f64::from(x) * f64::from(x)) + .sum::() + .sqrt(); + if !(norm.is_finite() && norm > 0.0) { + return Err(RealizarError::InferenceError(format!( + "pooled embedding has length {norm}: cannot scale it to unit length" + ))); + } + let inv = (1.0 / norm) as f32; + for x in &mut v { + *x *= inv; + } + Ok(v) +} + +/// The first `dim` values of a pooled vector, scaled back to unit length (Matryoshka). +/// +/// # Errors +/// A `dim` outside [`MATRYOSHKA_DIMS`] or wider than the vector, or a prefix of zero length. +pub fn truncate_to(vector: &[f32], dim: usize) -> Result> { + if !MATRYOSHKA_DIMS.contains(&dim) { + return Err(refuse(format!( + "--dim {dim}; the model is trained at {MATRYOSHKA_DIMS:?}" + ))); + } + let prefix = vector.get(..dim).ok_or_else(|| { + refuse(format!( + "--dim {dim} is wider than the {}-wide vector", + vector.len() + )) + })?; + unit_length(prefix.to_vec()) +} + +#[cfg(test)] +#[path = "forward_embedding_gemma2_tests.rs"] +mod tests; diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2_tests.rs new file mode 100644 index 0000000000..352e4876a9 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_embedding_gemma2_tests.rs @@ -0,0 +1,596 @@ +//! Tests for the EmbeddingGemma 2 CPU forward (APR-EMBED-001 EG-2, #4901). +//! +//! A synthetic F32 `gemma-embedding2` file small enough to run in a unit test: hidden 8, +//! per-layer input 4, two layers (0 local with head width 2, 1 global with head width 4), +//! two query heads over one KV head, FFN 8, output 6, vocabulary 5, window 2. +//! +//! Tests cover: +//! - The output: unit length, `output_dim()` wide, deterministic, order-sensitive (RoPE) +//! - The fixture's embeddings span more than a plane, so the reach test has room to see a move +//! - The fixture holds each weight's shape as a real file does, `[out, in]` once read +//! - Every tensor the graph loads reaches the embedding +//! - The last layer's output scale cancels in `output_norm` (the scale sits before it) +//! - Refusals: architecture, pooling, causal, RoPE scaling, shared KV, tensor count, +//! a missing tensor, wrong dims, the sliding-window pattern, head widths, inputs +//! - Token rows of a quantized table, and a row that is not whole blocks +//! - Matryoshka truncation + +use super::*; +use crate::gguf::test_factory::GGUFBuilder; + +const ARCH: &str = "gemma-embedding2"; +const H: usize = 8; +const P: usize = 4; +const LAYERS: usize = 2; +const HEADS: usize = 2; +const FF: usize = 8; +const OUT: usize = 6; +const VOCAB: usize = 5; +/// Head width of layer 0 (local) and layer 1 (global). +const HEAD_DIMS: [usize; LAYERS] = [2, 4]; +const TOKENS: [u32; 4] = [1, 2, 3, 4]; + +#[derive(Debug, Clone)] +enum Meta { + U32(u32), + F32(f32), + Bool(bool), + Str(&'static str), + I32s(Vec), + Bools(Vec), +} + +/// The valid synthetic file, with knobs a refusal test turns. +#[derive(Debug, Clone)] +struct Fixture { + arch: &'static str, + meta: Vec<(String, Meta)>, + tensors: Vec<(String, Vec, Vec)>, +} + +/// Deterministic values in `[-scale, scale)`: no RNG, so a failure reproduces exactly. An +/// integer hash of the index, not a sinusoid of it: `sin(a·i + b)` along a matrix's flattened +/// index is `sin(A_row + B_col)`, which is rank 2, and with every weight rank 2 the whole +/// fixture embedded onto one circle, where a weight could move the vector by less than the +/// cosine can resolve. +fn ramp(n: usize, seed: f32, scale: f32) -> Vec { + (0u32..) + .take(n) + .map(|i| { + let k = (i ^ seed.to_bits()).wrapping_mul(0x9E37_79B9); + let k = (k ^ (k >> 16)).wrapping_mul(0x85EB_CA6B); + let k = k ^ (k >> 13); + ((k >> 8) as f32 / 16_777_216.0 * 2.0 - 1.0) * scale + }) + .collect() +} + +/// A norm weight near 1. +fn norm(n: usize, seed: f32) -> Vec { + ramp(n, seed, 0.1).into_iter().map(|v| v + 1.0).collect() +} + +impl Fixture { + fn new() -> Self { + let mut f = Self { + arch: ARCH, + meta: Vec::new(), + tensors: Vec::new(), + }; + for (suffix, value) in [ + ("embedding_length", Meta::U32(H as u32)), + ("embedding_length_per_layer_input", Meta::U32(P as u32)), + ("embedding_length_out", Meta::U32(OUT as u32)), + ("block_count", Meta::U32(LAYERS as u32)), + ("feed_forward_length", Meta::U32(FF as u32)), + ("context_length", Meta::U32(16)), + ("pooling_type", Meta::U32(1)), + ("attention.head_count", Meta::U32(HEADS as u32)), + ("attention.head_count_kv", Meta::I32s(vec![1, 1])), + ("attention.causal", Meta::Bool(false)), + ("attention.shared_kv_layers", Meta::U32(0)), + ("attention.layer_norm_rms_epsilon", Meta::F32(1e-6)), + ("attention.sliding_window", Meta::U32(2)), + ( + "attention.sliding_window_pattern", + Meta::Bools(vec![true, false]), + ), + ("attention.key_length", Meta::U32(4)), + ("attention.value_length", Meta::U32(4)), + ("attention.key_length_swa", Meta::U32(2)), + ("attention.value_length_swa", Meta::U32(2)), + ("rope.dimension_count", Meta::U32(4)), + ("rope.dimension_count_swa", Meta::U32(2)), + ("rope.freq_base", Meta::F32(1e6)), + ("rope.freq_base_swa", Meta::F32(1e4)), + ] { + f = f.set(suffix, value); + } + let mut seed = 0.0; + let mut next = || { + seed += 1.7; + seed + }; + f = f + .weight("token_embd.weight", H, VOCAB, next()) + .weight("per_layer_model_proj.weight", H, P * LAYERS, next()) + .vector("per_layer_proj_norm.weight", P, next()) + .vector("output_norm.weight", H, next()) + .weight("output.weight", H, OUT, next()); + for (i, d) in HEAD_DIMS.into_iter().enumerate() { + let blk = |t: &str| format!("blk.{i}.{t}.weight"); + f = f + .vector(&blk("attn_norm"), H, next()) + .weight(&blk("attn_q"), H, HEADS * d, next()) + .weight(&blk("attn_k"), H, d, next()) + .weight(&blk("attn_v"), H, d, next()) + .weight(&blk("attn_output"), HEADS * d, H, next()) + .vector(&blk("attn_q_norm"), d, next()) + .vector(&blk("attn_k_norm"), d, next()) + .vector(&blk("post_attention_norm"), H, next()) + .vector(&blk("ffn_norm"), H, next()) + .weight(&blk("ffn_gate"), H, FF, next()) + .weight(&blk("ffn_up"), H, FF, next()) + .weight(&blk("ffn_down"), FF, H, next()) + .vector(&blk("post_ffw_norm"), H, next()) + .weight(&blk("inp_gate"), H, P, next()) + .weight(&blk("proj"), P, H, next()) + .vector(&blk("post_norm"), H, next()) + .tensor(&blk("layer_output_scale"), &[1], vec![0.9]); + } + f + } + + fn set(mut self, suffix: &str, value: Meta) -> Self { + let key = format!("{ARCH}.{suffix}"); + self.meta.retain(|(k, _)| *k != key); + self.meta.push((key, value)); + self + } + + fn unset(mut self, suffix: &str) -> Self { + let key = format!("{ARCH}.{suffix}"); + self.meta.retain(|(k, _)| *k != key); + self + } + + fn tensor(mut self, name: &str, dims: &[u64], data: Vec) -> Self { + self.tensors.retain(|(n, _, _)| n != name); + self.tensors.push((name.to_string(), dims.to_vec(), data)); + self + } + + /// `out_dim` rows of `in_dim`. The builder takes the shape row-major, `[out_dim, in_dim]`, + /// and writes it reversed, as GGUF's `ne0, ne1`, which is how a real file holds it. + fn weight(self, name: &str, in_dim: usize, out_dim: usize, seed: f32) -> Self { + let dims = [out_dim as u64, in_dim as u64]; + self.tensor(name, &dims, ramp(in_dim * out_dim, seed, 0.3)) + } + + fn vector(self, name: &str, len: usize, seed: f32) -> Self { + self.tensor(name, &[len as u64], norm(len, seed)) + } + + fn rename(mut self, from: &str, to: &str) -> Self { + for t in &mut self.tensors { + if t.0 == from { + t.0 = to.to_string(); + } + } + self + } + + fn bytes(&self) -> Vec { + let mut b = GGUFBuilder::new().architecture(self.arch); + for (key, value) in &self.meta { + b = match value { + Meta::U32(v) => b.add_u32(key, *v), + Meta::F32(v) => b.add_f32(key, *v), + Meta::Bool(v) => b.add_bool(key, *v), + Meta::Str(v) => b.add_string(key, v), + Meta::I32s(v) => b.add_i32_array(key, v), + Meta::Bools(v) => b.add_bool_array(key, v), + }; + } + for (name, dims, data) in &self.tensors { + b = b.add_f32_tensor(name, dims, data); + } + b.build() + } + + fn load(&self) -> Result { + let data = self.bytes(); + let model = GGUFModel::from_bytes(&data)?; + EmbeddingGemma2::from_gguf(&model, &data) + } + + fn embed(&self) -> Vec { + self.load() + .expect("the fixture loads") + .embed(&TOKENS) + .expect("the fixture embeds") + } + + /// The refusal reason, which must name `needle`. + fn refused(&self, needle: &str) { + let err = self.load().expect_err("the file must be refused"); + let text = err.to_string(); + assert!( + text.contains(needle), + "refusal {text:?} does not name {needle:?}" + ); + } +} + +fn length(v: &[f32]) -> f32 { + v.iter().map(|x| x * x).sum::().sqrt() +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum::() / (length(a) * length(b)) +} + +// ============================================================================ +// The output +// ============================================================================ + +#[test] +fn embeds_to_a_unit_vector_of_the_output_width_eg2() { + let model = Fixture::new().load().expect("the fixture loads"); + assert_eq!(model.output_dim(), OUT); + assert_eq!(model.vocab_size(), VOCAB); + let v = model.embed(&TOKENS).expect("embeds"); + assert_eq!(v.len(), OUT); + assert!(v.iter().all(|x| x.is_finite()), "{v:?}"); + assert!((length(&v) - 1.0).abs() < 1e-5, "length {}", length(&v)); + // A single token is a valid input too. + assert_eq!(model.embed(&[0]).expect("one token").len(), OUT); +} + +#[test] +fn the_same_tokens_give_the_same_vector_eg2() { + let model = Fixture::new().load().expect("the fixture loads"); + assert_eq!( + model.embed(&TOKENS).expect("embeds"), + model.embed(&TOKENS).expect("embeds") + ); +} + +/// Mean pooling over a non-causal stack is order-blind without positions; RoPE is what makes +/// the order count. +#[test] +fn token_order_changes_the_vector_eg2() { + let model = Fixture::new().load().expect("the fixture loads"); + let forward = model.embed(&[1, 2, 3, 4]).expect("embeds"); + let reversed = model.embed(&[4, 3, 2, 1]).expect("embeds"); + assert!( + cosine(&forward, &reversed) < 0.999_99, + "order ignored: {forward:?}" + ); +} + +/// The reach test below needs room to move: three embeddings of the fixture must span three +/// dimensions. With rank-2 weights they lay in one plane and the Gram determinant was 0. +#[test] +fn the_fixture_embeds_beyond_a_plane_eg2() { + let model = Fixture::new().load().expect("the fixture loads"); + let inputs: [&[u32]; 3] = [&[1, 2, 3, 4], &[4, 3, 2, 1], &[0, 2]]; + let [a, b, c] = inputs.map(|t| model.embed(t).expect("embeds")); + let dot = |x: &[f32], y: &[f32]| -> f64 { + x.iter() + .zip(y) + .map(|(p, q)| f64::from(*p) * f64::from(*q)) + .sum() + }; + let (ab, bc, ca) = (dot(&a, &b), dot(&b, &c), dot(&c, &a)); + let gram = dot(&a, &a) * dot(&b, &b) * dot(&c, &c) + 2.0 * ab * bc * ca + - ab * ab * dot(&c, &c) + - bc * bc * dot(&a, &a) + - ca * ca * dot(&b, &b); + assert!(gram > 1e-4, "coplanar embeddings: Gram determinant {gram}"); +} + +/// A weight that is loaded but never applied would survive this: each tensor is perturbed in +/// turn, non-uniformly, and the vector must move. +#[test] +fn every_tensor_reaches_the_embedding_eg2() { + let base = Fixture::new(); + let reference = base.embed(); + // The last layer's output scale multiplies the whole stream just before `output_norm`, + // which divides it back out; the next test pins that. + let cancels = format!("blk.{}.layer_output_scale.weight", LAYERS - 1); + let unreached: Vec = base + .tensors + .iter() + .filter(|(name, _, _)| *name != cancels) + .filter_map(|(name, dims, data)| { + let moved: Vec = data + .iter() + .enumerate() + .map(|(i, v)| v + 0.3 * (i as f32 * 1.3).cos() + 0.2) + .collect(); + let c = cosine(&reference, &base.clone().tensor(name, dims, moved).embed()); + (c >= 0.999_999).then(|| format!("{name} (cosine {c})")) + }) + .collect(); + assert!( + unreached.is_empty(), + "did not reach the embedding: {unreached:?}" + ); +} + +#[test] +fn the_last_layer_scale_cancels_in_the_output_norm_eg2() { + let base = Fixture::new(); + let name = format!("blk.{}.layer_output_scale.weight", LAYERS - 1); + let scaled = base.clone().tensor(&name, &[1], vec![3.0]).embed(); + assert!(cosine(&base.embed(), &scaled) > 0.999_99); + // The first layer's scale does not cancel: layer 1 adds its branches to the scaled stream. + let first = base + .clone() + .tensor("blk.0.layer_output_scale.weight", &[1], vec![3.0]) + .embed(); + assert!(cosine(&base.embed(), &first) < 0.999_99); +} + +// ============================================================================ +// Refusals: the file +// ============================================================================ + +#[test] +fn another_architecture_is_refused_eg2() { + let mut f = Fixture::new(); + f.arch = "gemma3"; + f.refused("is not gemma-embedding2"); +} + +#[test] +fn a_pooling_type_other_than_mean_is_refused_eg2() { + Fixture::new() + .set("pooling_type", Meta::U32(2)) + .refused("pooling_type 2"); + Fixture::new() + .unset("pooling_type") + .refused("pooling_type missing"); +} + +#[test] +fn a_causal_flag_is_refused_eg2() { + Fixture::new() + .set("attention.causal", Meta::Bool(true)) + .refused("causal"); + // Absent means not causal, as llama.cpp reads it. + assert!(Fixture::new().unset("attention.causal").load().is_ok()); +} + +#[test] +fn rope_scaling_and_shared_kv_are_refused_eg2() { + Fixture::new() + .set("rope.scaling.type", Meta::Str("yarn")) + .refused("rope.scaling.type"); + Fixture::new() + .set("attention.shared_kv_layers", Meta::U32(1)) + .refused("shared_kv_layers"); + assert!(Fixture::new() + .unset("attention.shared_kv_layers") + .load() + .is_ok()); +} + +#[test] +fn a_tensor_count_other_than_the_graph_is_refused_eg2() { + Fixture::new() + .tensor("extra.weight", &[H as u64], norm(H, 0.0)) + .refused("the graph consumes 39"); + Fixture::new() + .rename("blk.1.proj.weight", "blk.1.projx.weight") + .refused("blk.1.proj.weight missing"); +} + +/// The reader reverses GGUF's `ne0, ne1`, so a weight reads back as `[out, in]`: `token_embd` as +/// `[vocab, hidden]`, where the generative config reads its vocabulary (C-13), and `attn_q` as +/// `[heads × head width, hidden]`. A fixture written the other way round once loaded a forward that +/// read the other way round, and only the real file showed it. +#[test] +fn the_fixture_holds_shapes_as_a_real_file_does_eg2() { + let data = Fixture::new().bytes(); + let model = GGUFModel::from_bytes(&data).expect("parses"); + let dims = |name: &str| tensor_dims(&model, name).expect("the fixture has it"); + assert_eq!(dims("token_embd.weight"), [VOCAB, H]); + for (i, d) in HEAD_DIMS.into_iter().enumerate() { + assert_eq!(dims(&format!("blk.{i}.attn_q.weight")), [HEADS * d, H]); + } +} + +#[test] +fn a_tensor_with_other_dims_is_refused_eg2() { + Fixture::new() + .weight("blk.0.ffn_up.weight", H, FF + 1, 0.5) + .refused("blk.0.ffn_up.weight has dims [9, 8], expected [8, 8]"); + Fixture::new() + .tensor( + "output.weight", + &[H as u64, OUT as u64], + ramp(H * OUT, 0.5, 0.3), + ) + .refused("output.weight has dims [8, 6], expected [6, 8]"); + Fixture::new() + .tensor("blk.1.attn_q_norm.weight", &[2], norm(2, 0.5)) + .refused("blk.1.attn_q_norm.weight has dims [2], expected [4]"); +} + +#[test] +fn a_sliding_window_pattern_that_disagrees_is_refused_eg2() { + let key = "attention.sliding_window_pattern"; + Fixture::new() + .set(key, Meta::Bools(vec![false, true])) + .refused("layer 0"); + Fixture::new() + .set(key, Meta::Bools(vec![true])) + .refused("1 entries for 2 layers"); + Fixture::new() + .set(key, Meta::I32s(vec![1, 0])) + .refused("non-bool"); + // Absent, llama.cpp's default makes both layers local; the widths say layer 1 is global. + Fixture::new().unset(key).refused("layer 1"); +} + +#[test] +fn head_widths_that_do_not_fit_are_refused_eg2() { + Fixture::new() + .set("attention.value_length_swa", Meta::U32(3)) + .refused("value_length_swa 3"); + Fixture::new() + .set("rope.dimension_count", Meta::U32(3)) + .refused("rope.dimension_count 3"); + Fixture::new() + .set("rope.dimension_count_swa", Meta::U32(4)) + .refused("1..=2"); + Fixture::new() + .set("attention.head_count_kv", Meta::I32s(vec![1, 3])) + .refused("2 query heads over 3 KV heads"); + Fixture::new() + .set("embedding_length_out", Meta::U32(0)) + .refused("a zero width"); +} + +// ============================================================================ +// Refusals: the input +// ============================================================================ + +#[test] +fn inputs_the_model_cannot_embed_are_refused_eg2() { + let model = Fixture::new().load().expect("the fixture loads"); + let empty = model.embed(&[]).expect_err("no tokens"); + assert!(empty.to_string().contains("no tokens"), "{empty}"); + let oov = model + .embed(&[1, VOCAB as u32]) + .expect_err("out of vocabulary"); + assert!(oov.to_string().contains("outside the vocabulary"), "{oov}"); + let long = model.embed(&[1; 17]).expect_err("past the context"); + assert!( + matches!( + long, + RealizarError::ContextLimitExceeded { + provided: 17, + maximum: 16 + } + ), + "{long:?}" + ); + assert!(model.embed(&[1; 16]).is_ok(), "the full context fits"); +} + +// ============================================================================ +// Token rows and block sizes +// ============================================================================ + +/// Two rows of one Q4_K block each: `d` = 1.0, `dmin` = 0, varied scales and nibbles. +fn q4_k_rows(rows: usize) -> Vec { + let mut bytes = Vec::with_capacity(rows * 144); + for r in 0..rows { + bytes.extend_from_slice(&0x3C00u16.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend((0..12).map(|i| (i * 5 + r * 3 + 1) as u8 & 0x3F)); + bytes.extend((0..128).map(|i| (i * 7 + r * 11) as u8)); + } + bytes +} + +#[test] +fn a_quantized_token_row_matches_the_whole_table_eg2() { + let data = GGUFBuilder::new() + .architecture(ARCH) + .add_q4_k_tensor("t.weight", &[2, 256], &q4_k_rows(2)) + .build(); + let model = GGUFModel::from_bytes(&data).expect("parses"); + let table = weight(&model, &data, "t.weight", 256, 2).expect("whole blocks"); + let all = model + .get_tensor_f32("t.weight", &data) + .expect("dequantizes"); + assert_eq!(table_row(&table, 0).expect("row 0"), all[..256]); + assert_eq!(table_row(&table, 1).expect("row 1"), all[256..]); + assert_ne!( + all[..256], + all[256..], + "the rows must differ for the test to bite" + ); + assert!(table_row(&table, 2).is_err(), "row 2 of 2"); +} + +#[test] +fn a_row_that_is_not_whole_blocks_is_refused_eg2() { + let data = GGUFBuilder::new() + .architecture(ARCH) + .add_q4_k_tensor("t.weight", &[2, 128], &q4_k_rows(1)) + .build(); + let model = GGUFModel::from_bytes(&data).expect("parses"); + let err = weight(&model, &data, "t.weight", 128, 2).expect_err("half a block per row"); + assert!( + err.to_string().contains("not whole Q4_K blocks of 256"), + "{err}" + ); +} + +#[test] +fn bool_arrays_round_trip_through_the_reader_eg2() { + let data = GGUFBuilder::new() + .architecture(ARCH) + .add_bool_array("k", &[true, false, true]) + .build(); + let model = GGUFModel::from_bytes(&data).expect("parses"); + let want = GGUFValue::Array(vec![ + GGUFValue::Bool(true), + GGUFValue::Bool(false), + GGUFValue::Bool(true), + ]); + assert_eq!(model.metadata.get("k"), Some(&want)); +} + +// ============================================================================ +// Matryoshka truncation and pooling +// ============================================================================ + +#[test] +fn truncation_keeps_the_prefix_direction_at_unit_length_eg2() { + let v = unit_length(ramp(768, 0.5, 1.0)).expect("non-zero"); + for dim in MATRYOSHKA_DIMS { + let t = truncate_to(&v, dim).expect("a trained width"); + assert_eq!(t.len(), dim); + assert!((length(&t) - 1.0).abs() < 1e-5); + assert!(cosine(&t, &v[..dim]) > 0.999_999); + } + // At the full width the vector is scaled by 1 / |v|, which is 1 to within rounding. + let full = truncate_to(&v, 768).expect("full width"); + let moved = full + .iter() + .zip(&v) + .map(|(a, b)| (a - b).abs()) + .fold(0.0, f32::max); + assert!(moved <= 1e-6, "full width moved the vector by {moved}"); +} + +#[test] +fn truncation_to_an_untrained_or_wider_width_is_refused_eg2() { + let v = unit_length(ramp(768, 0.5, 1.0)).expect("non-zero"); + for dim in [0, 100, 1024] { + assert!(truncate_to(&v, dim).is_err(), "dim {dim}"); + } + assert!( + truncate_to(&v[..512], 768).is_err(), + "wider than the vector" + ); + assert!( + truncate_to(&[0.0; 768], 128).is_err(), + "a zero prefix has no direction" + ); +} + +#[test] +fn pooling_is_the_mean_over_tokens_eg2() { + assert_eq!(mean_rows(&[1.0, 2.0, 3.0, 6.0], 2), [2.0, 4.0]); + assert!(unit_length(vec![0.0; 3]).is_err()); + assert!(unit_length(vec![f32::NAN, 1.0]).is_err()); + let u = unit_length(vec![3.0, 4.0]).expect("non-zero"); + assert!((u[0] - 0.6).abs() < 1e-7 && (u[1] - 0.8).abs() < 1e-7); +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/mod.rs b/crates/aprender-serve/src/gguf/inference/forward/mod.rs index 82f40f3d6f..d5ca19ed20 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/mod.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/mod.rs @@ -18,6 +18,8 @@ mod traced; // PMAT-395: Re-export encoder-decoder types pub use encoder_decoder::EncoderOutput; +#[cfg(test)] +mod attention_windowed_tests; #[cfg(test)] mod batch_tests; #[cfg(test)] diff --git a/crates/aprender-serve/src/gguf/inference/fused_matmul_into.rs b/crates/aprender-serve/src/gguf/inference/fused_matmul_into.rs index d0854fbcd9..a71d21877e 100644 --- a/crates/aprender-serve/src/gguf/inference/fused_matmul_into.rs +++ b/crates/aprender-serve/src/gguf/inference/fused_matmul_into.rs @@ -79,7 +79,6 @@ impl OwnedQuantizedModel { /// `fused_matmul`, so that function's complexity does not rise (the /// per-function ratchet is shrink-only). pub(crate) fn dequant_fallback_or_refuse( - &self, input: &[f32], weight: &OwnedQuantizedTensor, in_dim: usize, @@ -98,7 +97,7 @@ impl OwnedQuantizedModel { ), }); } - self.dequant_fallback_matmul(input, weight, in_dim, out_dim, seq_len) + Self::dequant_fallback_matmul(input, weight, in_dim, out_dim, seq_len) } /// Dequantize-then-dot CPU path for the types with no fused kernel here @@ -110,7 +109,6 @@ impl OwnedQuantizedModel { /// here, not fast: one row is dequantized into a 256-element scratch at a /// time and dotted with the activation. pub(crate) fn dequant_fallback_matmul( - &self, input: &[f32], weight: &OwnedQuantizedTensor, in_dim: usize, diff --git a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs index a94f9737ee..a441479d42 100644 --- a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs +++ b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs @@ -136,6 +136,17 @@ impl OwnedQuantizedModel { input: &[f32], weight: &OwnedQuantizedTensor, ) -> Result> { + Self::matmul_any(input, weight) + } + + /// The one CPU quant dispatcher behind [`Self::fused_matmul`], as an associated + /// function: it reads no model field, so a forward that is not an + /// `OwnedQuantizedModel` (APR-EMBED-001 EG-2, the embedding forward) runs the same + /// arms instead of a second quant match (I-10). + /// + /// # Errors + /// An empty or undersized weight buffer, or a type with no arm here. + pub(crate) fn matmul_any(input: &[f32], weight: &OwnedQuantizedTensor) -> Result> { use crate::quantize::{dequantize_q4_1, dequantize_q5_0, dequantize_q5_1}; let in_dim = weight.in_dim; @@ -167,7 +178,7 @@ impl OwnedQuantizedModel { // arm, the IQ*/Q2_K/Q3_K dequant fallback (PMAT-3477 / #3091) let data = &weight.data; match weight.qtype { - GGUF_TYPE_F32 => Ok(self.fused_matmul_f32(input, data, in_dim, out_dim, seq_len)), + GGUF_TYPE_F32 => Ok(Self::fused_matmul_f32(input, data, in_dim, out_dim, seq_len)), GGUF_TYPE_BF16 => Ok(float16_matmul( input, data, @@ -185,7 +196,7 @@ impl OwnedQuantizedModel { Float16Kind::F16, )), GGUF_TYPE_Q4_0 | GGUF_TYPE_Q8_0 => { - self.fused_matmul_q4_q8(input, weight, in_dim, out_dim, seq_len) + Self::fused_matmul_q4_q8(input, weight, in_dim, out_dim, seq_len) }, GGUF_TYPE_Q4_1 => dequant_f32_matmul( input, @@ -230,13 +241,12 @@ impl OwnedQuantizedModel { let w = crate::apr::dequant::dequantize_apr_q8(data, in_dim * out_dim); dequant_f32_matmul(input, w, "APR-Q8", in_dim, out_dim, seq_len) }, - _ => self.fused_matmul_k_quants(input, weight, in_dim, out_dim, seq_len), + _ => Self::fused_matmul_k_quants(input, weight, in_dim, out_dim, seq_len), } } /// F32 zero-copy rayon matmul (extracted for complexity) fn fused_matmul_f32( - &self, input: &[f32], data: &[u8], in_dim: usize, @@ -307,7 +317,6 @@ impl OwnedQuantizedModel { /// Fused integer SIMD matmul for Q4_0 and Q8_0 fn fused_matmul_q4_q8( - &self, input: &[f32], weight: &OwnedQuantizedTensor, in_dim: usize, @@ -336,7 +345,6 @@ impl OwnedQuantizedModel { /// Fused K-quant kernels for Q4_K, Q5_K, Q6_K fn fused_matmul_k_quants( - &self, input: &[f32], weight: &OwnedQuantizedTensor, in_dim: usize, @@ -358,7 +366,7 @@ impl OwnedQuantizedModel { // PMAT-3477 / #3091: the IQ formats real unsloth GGUFs ship (and // Q2_K/Q3_K) take the dequantize-then-dot path; anything else is // refused by name, in the same default arm as before. - _ => self.dequant_fallback_or_refuse(x, weight, in_dim, out_dim, 1)?, + _ => Self::dequant_fallback_or_refuse(x, weight, in_dim, out_dim, 1)?, }; output.extend_from_slice(&row_output); } @@ -368,7 +376,7 @@ impl OwnedQuantizedModel { GGUF_TYPE_Q4_K => fused_q4k_parallel_matvec(&weight.data, input, in_dim, out_dim), GGUF_TYPE_Q5_K => fused_q5k_parallel_matvec(&weight.data, input, in_dim, out_dim), GGUF_TYPE_Q6_K => fused_q6k_parallel_matvec(&weight.data, input, in_dim, out_dim), - _ => self.dequant_fallback_or_refuse(input, weight, in_dim, out_dim, 1), + _ => Self::dequant_fallback_or_refuse(input, weight, in_dim, out_dim, 1), } } } diff --git a/crates/aprender-serve/src/gguf/metadata.rs b/crates/aprender-serve/src/gguf/metadata.rs index f4a0ec5129..bf976e2cf7 100644 --- a/crates/aprender-serve/src/gguf/metadata.rs +++ b/crates/aprender-serve/src/gguf/metadata.rs @@ -54,7 +54,21 @@ impl GGUFModel { reason: format!("Offset {} exceeds platform usize limit", tensor.offset), })?; let offset = self.tensor_data_start + tensor_offset; + Self::dequantize_at(tensor.qtype, file_data, offset, size) + } + /// `size` elements of ggml type `qtype` stored at byte `offset` of `file_data`, dequantized + /// to f32 — the one type match behind [`Self::get_tensor_f32`]. APR-EMBED-001 EG-2 reads a + /// single row of a quantized token table through it (I-10: no second quant match). + /// + /// # Errors + /// A byte range outside `file_data`, or a type with no dequantizer here. + pub(crate) fn dequantize_at( + qtype: u32, + file_data: &[u8], + offset: usize, + size: usize, + ) -> Result> { // Extract and dequantize based on qtype. A block type reads whole blocks and trims the // padding the last block carries; every type checks its byte range against the file first. use crate::quantize::{ @@ -64,7 +78,7 @@ impl GGUFModel { }; type Dequant = fn(&[u8]) -> Result>; // (elements per block, bytes per block, dequantizer) - let blocked: Option<(usize, usize, Dequant)> = match tensor.qtype { + let blocked: Option<(usize, usize, Dequant)> = match qtype { GGUF_TYPE_Q4_0 => Some((32, 18, dequantize_q4_0)), GGUF_TYPE_Q8_0 => Some((32, 34, dequantize_q8_0_simd)), GGUF_TYPE_Q2_K => Some((QK_K, 84, dequantize_q2_k)), @@ -83,7 +97,7 @@ impl GGUFModel { values.truncate(size); return Ok(values); } - match tensor.qtype { + match qtype { GGUF_TYPE_F32 => Ok(tensor_byte_range(file_data, offset, size * 4)? .as_chunks::<4>() .0 @@ -102,7 +116,7 @@ impl GGUFModel { )?)), _ => Err(RealizarError::UnsupportedOperation { operation: "get_tensor_f32".to_string(), - reason: format!("Unsupported quantization type: {}", tensor.qtype), + reason: format!("Unsupported quantization type: {qtype}"), }), } } diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index dfae41a33d..12ea5bfe2b 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -131,6 +131,9 @@ mod quantized_tests; #[cfg(test)] mod tests; +/// APR-EMBED-001 EG-2 (#4901): bidirectional windowed attention (I-10). +#[path = "inference/forward/attention_windowed.rs"] +pub mod attention_windowed; /// The dense (llama/qwen2/qwen3/...) forward behind the one engine (#4268): /// `apr run`, `run --batch`, `chat` and `serve` drive a dense GGUF through /// [`crate::session::Session`] on the CPU or the CUDA backend. @@ -140,10 +143,16 @@ pub mod dense_session; #[cfg(feature = "cuda")] #[path = "inference/forward/dense_session_borrowed.rs"] pub mod dense_session_borrowed; +/// APR-EMBED-001 EG-2 (#4901): the task prompts and framed token ids of an embedding input. +#[path = "inference/forward/embedding_gemma2_input.rs"] +pub mod embedding_gemma2_input; /// #3604: the F2 hybrid guard's receipt. CUDA-free on purpose, so its decision /// table is tested on every build. #[path = "inference/forward/f2_receipt.rs"] pub mod f2_receipt; +/// APR-EMBED-001 EG-2 (#4901): the EmbeddingGemma 2 CPU forward. +#[path = "inference/forward/forward_embedding_gemma2.rs"] +pub mod forward_embedding_gemma2; /// Qwen3.5 / Qwen3.8 hybrid (Gated `DeltaNet` + gated attention) CPU forward (#3091). #[path = "inference/forward/forward_qwen35.rs"] pub mod forward_qwen35; diff --git a/crates/aprender-serve/src/gguf/test_factory.rs b/crates/aprender-serve/src/gguf/test_factory.rs index 6d68755918..9249d36f81 100644 --- a/crates/aprender-serve/src/gguf/test_factory.rs +++ b/crates/aprender-serve/src/gguf/test_factory.rs @@ -425,6 +425,18 @@ impl GGUFBuilder { self } + /// Add a bool array metadata value (type 9, element_type 7), e.g. + /// `gemma-embedding2.attention.sliding_window_pattern` + #[must_use] + pub fn add_bool_array(mut self, key: &str, values: &[bool]) -> Self { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&7u32.to_le_bytes()); // element type = bool + bytes.extend_from_slice(&(values.len() as u64).to_le_bytes()); + bytes.extend(values.iter().map(|&v| u8::from(v))); + self.metadata.push((key.to_string(), 9, bytes)); + self + } + // ========================================================================= // Build // ========================================================================= diff --git a/docs/audits/APR-EMBED-001/EG-2/i8-cells.tsv b/docs/audits/APR-EMBED-001/EG-2/i8-cells.tsv new file mode 100644 index 0000000000..8ab6622034 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-2/i8-cells.tsv @@ -0,0 +1,841 @@ +file prompt length dim apr_tokens oracle_tokens oracle_width cosine max_abs_diff oracle_prompt +BF16 SearchQuery len00008 768 15 15 768 0.9999971085042846 0.0002769837390228999 equal +BF16 SearchQuery len00008 512 15 15 768 0.9999970232370413 0.00031981975325844417 equal +BF16 SearchQuery len00008 256 15 15 768 0.9999972849936736 0.00040206693721318854 equal +BF16 SearchQuery len00008 128 15 15 768 0.9999982197754205 0.00046775412657519555 equal +BF16 SearchQuery len00128 768 143 143 768 0.9999977152634244 0.00024874491638320756 equal +BF16 SearchQuery len00128 512 143 143 768 0.9999977421821822 0.0002875748698299013 equal +BF16 SearchQuery len00128 256 143 143 768 0.9999981395678417 0.00035242813572243514 equal +BF16 SearchQuery len00128 128 143 143 768 0.9999985510911035 0.0004261616474331986 equal +BF16 SearchQuery len01024 768 1148 1148 768 0.9999984192019972 0.00023470430461652522 equal +BF16 SearchQuery len01024 512 1148 1148 768 0.9999984715814074 0.0002671964246574038 equal +BF16 SearchQuery len01024 256 1148 1148 768 0.999998550117673 0.0003307198901140486 equal +BF16 SearchQuery len01024 128 1148 1148 768 0.9999989070137859 0.0003951557267063907 equal +BF16 SearchQuery len02000 768 2406 2406 768 0.9999982127214196 0.00022362804237232095 equal +BF16 SearchQuery len02000 512 2406 2406 768 0.9999982899835407 0.00025466070078079134 equal +BF16 SearchQuery len02000 256 2406 2406 768 0.9999984003752862 0.0003200345916376107 equal +BF16 SearchQuery len02000 128 2406 2406 768 0.9999989474036323 0.00036831087200798335 equal +BF16 SearchQuery len08000 768 11402 11402 768 0.9999992942216082 0.00012416530604361653 equal +BF16 SearchQuery len08000 512 11402 11402 768 0.9999993068425304 0.00014292032885575984 equal +BF16 SearchQuery len08000 256 11402 11402 768 0.9999993210463244 0.00017887541986481154 equal +BF16 SearchQuery len08000 128 11402 11402 768 0.9999995552207447 0.00020814839400461194 equal +BF16 QuestionAnswering len00008 768 15 15 768 0.9999954590569886 0.00034287680334411756 equal +BF16 QuestionAnswering len00008 512 15 15 768 0.9999953720704972 0.00039647998217319166 equal +BF16 QuestionAnswering len00008 256 15 15 768 0.9999958070142446 0.0004988619425600471 equal +BF16 QuestionAnswering len00008 128 15 15 768 0.9999973324996279 0.0004967653021677731 equal +BF16 QuestionAnswering len00128 768 143 143 768 0.9999965722369042 0.00034289295175755907 equal +BF16 QuestionAnswering len00128 512 143 143 768 0.9999967308519274 0.0003974573930611014 equal +BF16 QuestionAnswering len00128 256 143 143 768 0.9999970452072864 0.0004869668095540819 equal +BF16 QuestionAnswering len00128 128 143 143 768 0.9999982159305553 0.00038687221327656646 equal +BF16 QuestionAnswering len01024 768 1148 1148 768 0.9999961562511822 0.00033096798088369517 equal +BF16 QuestionAnswering len01024 512 1148 1148 768 0.9999961687877044 0.0003788730392027956 equal +BF16 QuestionAnswering len01024 256 1148 1148 768 0.9999966810873618 0.0004708720524236737 equal +BF16 QuestionAnswering len01024 128 1148 1148 768 0.9999975957239939 0.0004945886641972083 equal +BF16 QuestionAnswering len02000 768 2406 2406 768 0.999997510542839 0.0002729936437308178 equal +BF16 QuestionAnswering len02000 512 2406 2406 768 0.9999975707345057 0.00031286685236282114 equal +BF16 QuestionAnswering len02000 256 2406 2406 768 0.9999978431457612 0.0003877814618013231 equal +BF16 QuestionAnswering len02000 128 2406 2406 768 0.9999983385442474 0.00045611352952465434 equal +BF16 QuestionAnswering len08000 768 11402 11402 768 0.999999221370769 0.00013195187318627496 equal +BF16 QuestionAnswering len08000 512 11402 11402 768 0.9999992692881126 0.00014967250124273482 equal +BF16 QuestionAnswering len08000 256 11402 11402 768 0.9999992828896485 0.00018767875026620695 equal +BF16 QuestionAnswering len08000 128 11402 11402 768 0.9999995360225133 0.00021420475633282532 equal +BF16 FactChecking len00008 768 15 15 768 0.9999931930546878 0.00042563693358651546 equal +BF16 FactChecking len00008 512 15 15 768 0.9999931600585777 0.0004899355620683644 equal +BF16 FactChecking len00008 256 15 15 768 0.9999934330190088 0.0006171426009750416 equal +BF16 FactChecking len00008 128 15 15 768 0.9999958480930711 0.0006485922900802618 equal +BF16 FactChecking len00128 768 143 143 768 0.9999946902002964 0.00041974481833416746 equal +BF16 FactChecking len00128 512 143 143 768 0.9999949342872564 0.0004816045214197182 equal +BF16 FactChecking len00128 256 143 143 768 0.9999955782032034 0.0006018175594105987 equal +BF16 FactChecking len00128 128 143 143 768 0.9999968412335724 0.0007252136634883534 equal +BF16 FactChecking len01024 768 1148 1148 768 0.9999978485532481 0.00024907603463732164 equal +BF16 FactChecking len01024 512 1148 1148 768 0.9999979569073615 0.0002831057725450892 equal +BF16 FactChecking len01024 256 1148 1148 768 0.9999981868230161 0.0003525825052646675 equal +BF16 FactChecking len01024 128 1148 1148 768 0.9999987935507865 0.0004189470388342581 equal +BF16 FactChecking len02000 768 2406 2406 768 0.9999981375154626 0.00021961661034120353 equal +BF16 FactChecking len02000 512 2406 2406 768 0.9999981143275228 0.00025035161550284596 equal +BF16 FactChecking len02000 256 2406 2406 768 0.9999983717202866 0.00031041686928048873 equal +BF16 FactChecking len02000 128 2406 2406 768 0.9999989995226405 0.00034214010882618884 equal +BF16 FactChecking len08000 768 11402 11402 768 0.9999991607843198 0.00016114414432652686 equal +BF16 FactChecking len08000 512 11402 11402 768 0.9999991819444813 0.00018371614980220862 equal +BF16 FactChecking len08000 256 11402 11402 768 0.9999992307464957 0.00023226350979265245 equal +BF16 FactChecking len08000 128 11402 11402 768 0.9999994832782403 0.00023868660486325854 equal +BF16 CodeRetrieval len00008 768 15 15 768 0.9999964371806509 0.0003756999053051366 equal +BF16 CodeRetrieval len00008 512 15 15 768 0.9999964543328068 0.00043129764113569724 equal +BF16 CodeRetrieval len00008 256 15 15 768 0.999996801801278 0.0005375095731749713 equal +BF16 CodeRetrieval len00008 128 15 15 768 0.9999976457952531 0.0006093251015954759 equal +BF16 CodeRetrieval len00128 768 143 143 768 0.9999980879265906 0.00022015141593193302 equal +BF16 CodeRetrieval len00128 512 143 143 768 0.9999981698180626 0.0002533736758908757 equal +BF16 CodeRetrieval len00128 256 143 143 768 0.9999985174754104 0.0003150343491915321 equal +BF16 CodeRetrieval len00128 128 143 143 768 0.9999988403214125 0.0003176185728777786 equal +BF16 CodeRetrieval len01024 768 1148 1148 768 0.9999977007610967 0.00033648246730491604 equal +BF16 CodeRetrieval len01024 512 1148 1148 768 0.9999977056396026 0.00038472051405025534 equal +BF16 CodeRetrieval len01024 256 1148 1148 768 0.9999979764395712 0.0004808143290899805 equal +BF16 CodeRetrieval len01024 128 1148 1148 768 0.9999985620344056 0.000556774603025563 equal +BF16 CodeRetrieval len02000 768 2406 2406 768 0.9999968138394312 0.0003290932839836256 equal +BF16 CodeRetrieval len02000 512 2406 2406 768 0.9999969031001898 0.0003754013351855382 equal +BF16 CodeRetrieval len02000 256 2406 2406 768 0.9999969909256208 0.0004342688554886398 equal +BF16 CodeRetrieval len02000 128 2406 2406 768 0.9999979095711328 0.0004909403553764388 equal +BF16 CodeRetrieval len08000 768 11402 11402 768 0.9999992232167588 0.0001684738562047483 equal +BF16 CodeRetrieval len08000 512 11402 11402 768 0.9999992953524053 0.0001900153988163289 equal +BF16 CodeRetrieval len08000 256 11402 11402 768 0.9999994131377481 0.0002393896139691043 equal +BF16 CodeRetrieval len08000 128 11402 11402 768 0.9999996434891252 0.00027236847905680317 equal +BF16 Classification len00008 768 14 14 768 0.999990162586105 0.0005569244249444042 equal +BF16 Classification len00008 512 14 14 768 0.9999901041827565 0.0006422146530220658 equal +BF16 Classification len00008 256 14 14 768 0.9999909275140488 0.0007242479991074781 equal +BF16 Classification len00008 128 14 14 768 0.9999942354605045 0.0008266763164812285 equal +BF16 Classification len00128 768 142 142 768 0.9999979412088451 0.00020878573417749957 equal +BF16 Classification len00128 512 142 142 768 0.9999980403999003 0.00023913060926300522 equal +BF16 Classification len00128 256 142 142 768 0.9999984290457675 0.000291022726288781 equal +BF16 Classification len00128 128 142 142 768 0.999999019576877 0.00031986291494129696 equal +BF16 Classification len01024 768 1147 1147 768 0.9999981504327473 0.00026031379752104733 equal +BF16 Classification len01024 512 1147 1147 768 0.9999981189193311 0.00029695625391749914 equal +BF16 Classification len01024 256 1147 1147 768 0.999998262149146 0.000367752121810741 equal +BF16 Classification len01024 128 1147 1147 768 0.9999990240565195 0.00040082668456209447 equal +BF16 Classification len02000 768 2405 2405 768 0.9999974062651049 0.0003421131407613069 equal +BF16 Classification len02000 512 2405 2405 768 0.9999974054116036 0.0003903675775675311 equal +BF16 Classification len02000 256 2405 2405 768 0.9999975756771106 0.0004876486012556358 equal +BF16 Classification len02000 128 2405 2405 768 0.9999982740220651 0.0005527513580826995 equal +BF16 Classification len08000 768 11401 11401 768 0.9999991066824917 0.00016348060632923045 equal +BF16 Classification len08000 512 11401 11401 768 0.999999103382166 0.00018583759512869752 equal +BF16 Classification len08000 256 11401 11401 768 0.9999991180831145 0.00023522694291810975 equal +BF16 Classification len08000 128 11401 11401 768 0.9999994968347685 0.00025152974690939925 equal +BF16 Clustering len00008 768 14 14 768 0.9999982468014238 0.0002201574004792109 equal +BF16 Clustering len00008 512 14 14 768 0.9999982489163988 0.0002534068686886118 equal +BF16 Clustering len00008 256 14 14 768 0.9999984790329878 0.0003157230512183043 equal +BF16 Clustering len00008 128 14 14 768 0.9999991341919618 0.0003060892036907438 equal +BF16 Clustering len00128 768 142 142 768 0.999998457314099 0.00023097629352693966 equal +BF16 Clustering len00128 512 142 142 768 0.9999984784637598 0.00026474488015102576 equal +BF16 Clustering len00128 256 142 142 768 0.9999986318807552 0.00032240921491059893 equal +BF16 Clustering len00128 128 142 142 768 0.9999990714107521 0.00034808949308075656 equal +BF16 Clustering len01024 768 1147 1147 768 0.9999983147801131 0.000213321348273475 equal +BF16 Clustering len01024 512 1147 1147 768 0.9999983563641548 0.00024396872289127874 equal +BF16 Clustering len01024 256 1147 1147 768 0.999998509447901 0.00029988651739928235 equal +BF16 Clustering len01024 128 1147 1147 768 0.9999990960931299 0.00034334600007419197 equal +BF16 Clustering len02000 768 2405 2405 768 0.9999969493159457 0.00030476184354447794 equal +BF16 Clustering len02000 512 2405 2405 768 0.9999968908782133 0.0003472373476146734 equal +BF16 Clustering len02000 256 2405 2405 768 0.9999973332970932 0.0004286150717519241 equal +BF16 Clustering len02000 128 2405 2405 768 0.9999982888320506 0.00042862714129299206 equal +BF16 Clustering len08000 768 11401 11401 768 0.9999991401632894 0.0001440791152330905 equal +BF16 Clustering len08000 512 11401 11401 768 0.9999991778630664 0.00016388531460605205 equal +BF16 Clustering len08000 256 11401 11401 768 0.9999992190704945 0.0002058091611772568 equal +BF16 Clustering len08000 128 11401 11401 768 0.9999994773263339 0.00022951963286218027 equal +BF16 SentenceSimilarity len00008 768 15 15 768 0.9999983297593446 0.0002639842062540775 equal +BF16 SentenceSimilarity len00008 512 15 15 768 0.9999983872376471 0.0003040964532973233 equal +BF16 SentenceSimilarity len00008 256 15 15 768 0.9999986026660063 0.0003822085102380381 equal +BF16 SentenceSimilarity len00008 128 15 15 768 0.9999991393881634 0.0002863536229714275 equal +BF16 SentenceSimilarity len00128 768 143 143 768 0.9999988593792941 0.00019567438650551494 equal +BF16 SentenceSimilarity len00128 512 143 143 768 0.9999988853135129 0.00022561414534617852 equal +BF16 SentenceSimilarity len00128 256 143 143 768 0.9999989820030271 0.0002867872752776099 equal +BF16 SentenceSimilarity len00128 128 143 143 768 0.9999993023604291 0.000339104756960619 equal +BF16 SentenceSimilarity len01024 768 1148 1148 768 0.9999982409786591 0.000275942380029974 equal +BF16 SentenceSimilarity len01024 512 1148 1148 768 0.9999982858716294 0.00031648616050294204 equal +BF16 SentenceSimilarity len01024 256 1148 1148 768 0.9999984261074738 0.00039403669779872197 equal +BF16 SentenceSimilarity len01024 128 1148 1148 768 0.9999989538542499 0.0004513377715805039 equal +BF16 SentenceSimilarity len02000 768 2406 2406 768 0.9999984879936301 0.00024177626901119784 equal +BF16 SentenceSimilarity len02000 512 2406 2406 768 0.9999985311914016 0.00027619333838764526 equal +BF16 SentenceSimilarity len02000 256 2406 2406 768 0.9999986427984585 0.00031947873163268303 equal +BF16 SentenceSimilarity len02000 128 2406 2406 768 0.9999990793873953 0.00036851506818222357 equal +BF16 SentenceSimilarity len08000 768 11402 11402 768 0.9999993912112118 0.00012301080073680998 equal +BF16 SentenceSimilarity len08000 512 11402 11402 768 0.9999994219077758 0.0001400988908099622 equal +BF16 SentenceSimilarity len08000 256 11402 11402 768 0.9999994844108572 0.00016988646743384614 equal +BF16 SentenceSimilarity len08000 128 11402 11402 768 0.9999996787867564 0.0001944769847086296 equal +F16 SearchQuery len00008 768 15 15 768 0.9999999039383832 5.215399021352672e-05 equal +F16 SearchQuery len00008 512 15 15 768 0.9999999051826985 6.0322987472119505e-05 equal +F16 SearchQuery len00008 256 15 15 768 0.9999999103168042 7.520598302254711e-05 equal +F16 SearchQuery len00008 128 15 15 768 0.9999999465868596 7.796247570244705e-05 equal +F16 SearchQuery len00128 768 143 143 768 0.999999940357717 3.661159775768946e-05 equal +F16 SearchQuery len00128 512 143 143 768 0.9999999435649363 4.221007056735143e-05 equal +F16 SearchQuery len00128 256 143 143 768 0.9999999510273302 5.309748341815468e-05 equal +F16 SearchQuery len00128 128 143 143 768 0.9999999632188842 6.393210480715172e-05 equal +F16 SearchQuery len01024 768 1148 1148 768 0.9999999591577576 3.34566659035071e-05 equal +F16 SearchQuery len01024 512 1148 1148 768 0.9999999602598724 3.793503210256649e-05 equal +F16 SearchQuery len01024 256 1148 1148 768 0.9999999653846393 4.728370143235183e-05 equal +F16 SearchQuery len01024 128 1148 1148 768 0.9999999735905478 5.539738894551338e-05 equal +F16 SearchQuery len02000 768 2406 2406 768 0.9999999458380265 4.239439977634166e-05 equal +F16 SearchQuery len02000 512 2406 2406 768 0.999999947203367 4.842900290866897e-05 equal +F16 SearchQuery len02000 256 2406 2406 768 0.9999999488124038 6.027900369748318e-05 equal +F16 SearchQuery len02000 128 2406 2406 768 0.9999999660919483 7.019778111504518e-05 equal +F16 SearchQuery len08000 768 11402 11402 768 0.9999999754877015 2.7068663857612862e-05 equal +F16 SearchQuery len08000 512 11402 11402 768 0.9999999766036949 3.091416734397889e-05 equal +F16 SearchQuery len08000 256 11402 11402 768 0.9999999783283294 3.895195419039571e-05 equal +F16 SearchQuery len08000 128 11402 11402 768 0.9999999853863489 4.3416075756729997e-05 equal +F16 QuestionAnswering len00008 768 15 15 768 0.9999998766037056 5.410163424294373e-05 equal +F16 QuestionAnswering len00008 512 15 15 768 0.9999998748963023 6.223778135723557e-05 equal +F16 QuestionAnswering len00008 256 15 15 768 0.9999998794956235 7.4162685576553e-05 equal +F16 QuestionAnswering len00008 128 15 15 768 0.9999999156251772 8.626315586325768e-05 equal +F16 QuestionAnswering len00128 768 143 143 768 0.9999998298870894 7.441353287201274e-05 equal +F16 QuestionAnswering len00128 512 143 143 768 0.9999998344993138 8.595880872724893e-05 equal +F16 QuestionAnswering len00128 256 143 143 768 0.9999998547116289 0.00010480262303388943 equal +F16 QuestionAnswering len00128 128 143 143 768 0.9999998984839931 0.0001242252673612776 equal +F16 QuestionAnswering len01024 768 1148 1148 768 0.9999999310625545 4.431321396149591e-05 equal +F16 QuestionAnswering len01024 512 1148 1148 768 0.999999935665039 5.0559922939859454e-05 equal +F16 QuestionAnswering len01024 256 1148 1148 768 0.999999944877979 6.281296443874738e-05 equal +F16 QuestionAnswering len01024 128 1148 1148 768 0.9999999609528898 5.994238911892491e-05 equal +F16 QuestionAnswering len02000 768 2406 2406 768 0.9999999481113965 4.926892103722727e-05 equal +F16 QuestionAnswering len02000 512 2406 2406 768 0.9999999492672974 5.634743787247265e-05 equal +F16 QuestionAnswering len02000 256 2406 2406 768 0.9999999507144063 7.039629131579128e-05 equal +F16 QuestionAnswering len02000 128 2406 2406 768 0.9999999616976205 5.894073919995857e-05 equal +F16 QuestionAnswering len08000 768 11402 11402 768 0.9999999778063016 2.4065394473173074e-05 equal +F16 QuestionAnswering len08000 512 11402 11402 768 0.9999999795161002 2.7414929554370887e-05 equal +F16 QuestionAnswering len08000 256 11402 11402 768 0.9999999809174237 3.4401384617341324e-05 equal +F16 QuestionAnswering len08000 128 11402 11402 768 0.9999999887198683 3.95383843679719e-05 equal +F16 FactChecking len00008 768 15 15 768 0.9999994100042027 0.00013784041551482412 equal +F16 FactChecking len00008 512 15 15 768 0.9999994221149016 0.00016070013785238202 equal +F16 FactChecking len00008 256 15 15 768 0.9999994543414471 0.00020312565774516234 equal +F16 FactChecking len00008 128 15 15 768 0.9999996196624011 0.00022414112867799596 equal +F16 FactChecking len00128 768 143 143 768 0.9999997142901904 0.0001090441236301122 equal +F16 FactChecking len00128 512 143 143 768 0.9999997265383681 0.00012520381900182898 equal +F16 FactChecking len00128 256 143 143 768 0.9999997682071321 0.00015657555026817038 equal +F16 FactChecking len00128 128 143 143 768 0.9999998258999436 0.00018960345910751936 equal +F16 FactChecking len01024 768 1148 1148 768 0.999999918025984 5.040728042239942e-05 equal +F16 FactChecking len01024 512 1148 1148 768 0.9999999198371594 5.7485939513098744e-05 equal +F16 FactChecking len01024 256 1148 1148 768 0.9999999256740146 7.157136484728724e-05 equal +F16 FactChecking len01024 128 1148 1148 768 0.999999945718847 8.427912160592632e-05 equal +F16 FactChecking len02000 768 2406 2406 768 0.9999999372897469 4.227682862461158e-05 equal +F16 FactChecking len02000 512 2406 2406 768 0.9999999382504998 4.819643427872878e-05 equal +F16 FactChecking len02000 256 2406 2406 768 0.999999944613037 6.036566003517381e-05 equal +F16 FactChecking len02000 128 2406 2406 768 0.9999999688439491 6.29099147957269e-05 equal +F16 FactChecking len08000 768 11402 11402 768 0.9999999767984531 2.532352323225684e-05 equal +F16 FactChecking len08000 512 11402 11402 768 0.9999999776909699 2.7844597054954914e-05 equal +F16 FactChecking len08000 256 11402 11402 768 0.9999999792496945 3.52418378173798e-05 equal +F16 FactChecking len08000 128 11402 11402 768 0.9999999879708922 4.042269985646274e-05 equal +F16 CodeRetrieval len00008 768 15 15 768 0.999999909481374 4.925270335995411e-05 equal +F16 CodeRetrieval len00008 512 15 15 768 0.9999999126079848 5.6607089670606195e-05 equal +F16 CodeRetrieval len00008 256 15 15 768 0.9999999295980855 7.071845806423671e-05 equal +F16 CodeRetrieval len00008 128 15 15 768 0.9999999497248844 7.992278847540266e-05 equal +F16 CodeRetrieval len00128 768 143 143 768 0.9999999255341366 5.418968330655333e-05 equal +F16 CodeRetrieval len00128 512 143 143 768 0.9999999270993345 6.223154882040866e-05 equal +F16 CodeRetrieval len00128 256 143 143 768 0.9999999352539543 6.966670918960582e-05 equal +F16 CodeRetrieval len00128 128 143 143 768 0.9999999497185665 8.114248655390066e-05 equal +F16 CodeRetrieval len01024 768 1148 1148 768 0.9999999509953474 3.61444711916431e-05 equal +F16 CodeRetrieval len01024 512 1148 1148 768 0.9999999522773187 4.117919507600626e-05 equal +F16 CodeRetrieval len01024 256 1148 1148 768 0.9999999551760944 5.180716741923219e-05 equal +F16 CodeRetrieval len01024 128 1148 1148 768 0.9999999652184758 5.995514937091537e-05 equal +F16 CodeRetrieval len02000 768 2406 2406 768 0.9999999339463552 5.994173567492808e-05 equal +F16 CodeRetrieval len02000 512 2406 2406 768 0.9999999342585822 6.832744584728961e-05 equal +F16 CodeRetrieval len02000 256 2406 2406 768 0.9999999358856173 8.55335741694642e-05 equal +F16 CodeRetrieval len02000 128 2406 2406 768 0.9999999485584294 9.720558415585168e-05 equal +F16 CodeRetrieval len08000 768 11402 11402 768 0.9999999784272308 3.306938664230513e-05 equal +F16 CodeRetrieval len08000 512 11402 11402 768 0.9999999786607611 3.767909963099149e-05 equal +F16 CodeRetrieval len08000 256 11402 11402 768 0.9999999790707648 4.748379306178559e-05 equal +F16 CodeRetrieval len08000 128 11402 11402 768 0.9999999870899211 5.3943944334080424e-05 equal +F16 Classification len00008 768 14 14 768 0.9999997452120091 8.652141359948018e-05 equal +F16 Classification len00008 512 14 14 768 0.9999997410615549 9.956851339295442e-05 equal +F16 Classification len00008 256 14 14 768 0.9999997590139089 0.00012374576390737635 equal +F16 Classification len00008 128 14 14 768 0.9999998432937915 0.00014363023138823372 equal +F16 Classification len00128 768 142 142 768 0.9999999435487663 5.5459748176025364e-05 equal +F16 Classification len00128 512 142 142 768 0.9999999449693837 6.355164140598932e-05 equal +F16 Classification len00128 256 142 142 768 0.9999999480639794 7.874090543642451e-05 equal +F16 Classification len00128 128 142 142 768 0.9999999636378022 7.013296191548529e-05 equal +F16 Classification len01024 768 1147 1147 768 0.9999999457228098 4.067865183868605e-05 equal +F16 Classification len01024 512 1147 1147 768 0.9999999481415559 4.636757013600293e-05 equal +F16 Classification len01024 256 1147 1147 768 0.9999999559481458 5.772581841914995e-05 equal +F16 Classification len01024 128 1147 1147 768 0.9999999728121718 6.561605181915064e-05 equal +F16 Classification len02000 768 2405 2405 768 0.9999999389949674 4.764600609692693e-05 equal +F16 Classification len02000 512 2405 2405 768 0.999999941690126 5.423163758502131e-05 equal +F16 Classification len02000 256 2405 2405 768 0.9999999475836605 6.698300589357692e-05 equal +F16 Classification len02000 128 2405 2405 768 0.9999999658667472 7.538450351193826e-05 equal +F16 Classification len08000 768 11401 11401 768 0.9999999810458283 2.435398157784566e-05 equal +F16 Classification len08000 512 11401 11401 768 0.9999999816761892 2.7762197949101415e-05 equal +F16 Classification len08000 256 11401 11401 768 0.99999998249768 3.496807268658399e-05 equal +F16 Classification len08000 128 11401 11401 768 0.9999999872941194 3.804839774967407e-05 equal +F16 Clustering len00008 768 14 14 768 0.9999999579306407 3.2855096825157304e-05 equal +F16 Clustering len00008 512 14 14 768 0.9999999576459492 3.790932841066241e-05 equal +F16 Clustering len00008 256 14 14 768 0.999999961590936 4.718485637197206e-05 equal +F16 Clustering len00008 128 14 14 768 0.9999999730904927 5.3187251542810565e-05 equal +F16 Clustering len00128 768 142 142 768 0.9999999574812758 4.579670904709607e-05 equal +F16 Clustering len00128 512 142 142 768 0.9999999581117357 5.237234523458936e-05 equal +F16 Clustering len00128 256 142 142 768 0.9999999630373352 5.490919463191704e-05 equal +F16 Clustering len00128 128 142 142 768 0.9999999755989151 5.425216846363656e-05 equal +F16 Clustering len01024 768 1147 1147 768 0.9999999640856042 3.1752323300840435e-05 equal +F16 Clustering len01024 512 1147 1147 768 0.9999999640333481 3.625747839542531e-05 equal +F16 Clustering len01024 256 1147 1147 768 0.9999999682272208 4.058234675949346e-05 equal +F16 Clustering len01024 128 1147 1147 768 0.9999999798887871 4.574371055283272e-05 equal +F16 Clustering len02000 768 2405 2405 768 0.9999999306136237 8.089946891598476e-05 equal +F16 Clustering len02000 512 2405 2405 768 0.9999999320444452 9.328389775903334e-05 equal +F16 Clustering len02000 256 2405 2405 768 0.9999999398272681 0.00011395543630332572 equal +F16 Clustering len02000 128 2405 2405 768 0.9999999544811067 0.00012754947378323944 equal +F16 Clustering len08000 768 11401 11401 768 0.9999999803428498 2.166173927151214e-05 equal +F16 Clustering len08000 512 11401 11401 768 0.999999980938552 2.466045755403711e-05 equal +F16 Clustering len08000 256 11401 11401 768 0.9999999833050031 3.053780512843329e-05 equal +F16 Clustering len08000 128 11401 11401 768 0.999999988619316 3.116473626657568e-05 equal +F16 SentenceSimilarity len00008 768 15 15 768 0.9999999569172153 3.58880762232755e-05 equal +F16 SentenceSimilarity len00008 512 15 15 768 0.9999999575345532 4.1163161861458575e-05 equal +F16 SentenceSimilarity len00008 256 15 15 768 0.9999999613096582 5.157498011124778e-05 equal +F16 SentenceSimilarity len00008 128 15 15 768 0.9999999750538723 5.8934749862862845e-05 equal +F16 SentenceSimilarity len00128 768 143 143 768 0.9999999719701715 3.107133326721995e-05 equal +F16 SentenceSimilarity len00128 512 143 143 768 0.9999999720262613 3.581164957901749e-05 equal +F16 SentenceSimilarity len00128 256 143 143 768 0.999999974259153 4.4883325152463883e-05 equal +F16 SentenceSimilarity len00128 128 143 143 768 0.9999999805239081 4.809588011966265e-05 equal +F16 SentenceSimilarity len01024 768 1148 1148 768 0.9999999610593869 3.615402890252972e-05 equal +F16 SentenceSimilarity len01024 512 1148 1148 768 0.999999961171335 4.125508654600346e-05 equal +F16 SentenceSimilarity len01024 256 1148 1148 768 0.9999999650522321 5.183038909752774e-05 equal +F16 SentenceSimilarity len01024 128 1148 1148 768 0.9999999742171183 6.142646063452689e-05 equal +F16 SentenceSimilarity len02000 768 2406 2406 768 0.9999999618397792 3.222962225345183e-05 equal +F16 SentenceSimilarity len02000 512 2406 2406 768 0.9999999644922318 3.6807340063761684e-05 equal +F16 SentenceSimilarity len02000 256 2406 2406 768 0.999999968541312 4.607761992026296e-05 equal +F16 SentenceSimilarity len02000 128 2406 2406 768 0.9999999807346011 4.9942213217704645e-05 equal +F16 SentenceSimilarity len08000 768 11402 11402 768 0.9999999828447244 2.0742631889252537e-05 equal +F16 SentenceSimilarity len08000 512 11402 11402 768 0.9999999836104349 2.3644001881784407e-05 equal +F16 SentenceSimilarity len08000 256 11402 11402 768 0.9999999853556233 2.9909009118520014e-05 equal +F16 SentenceSimilarity len08000 128 11402 11402 768 0.9999999914421335 3.4050955628157215e-05 equal +Q8_0 SearchQuery len00008 768 15 15 768 0.9998831182106473 0.0016247440881461215 equal +Q8_0 SearchQuery len00008 512 15 15 768 0.999890804418292 0.001880307258343059 equal +Q8_0 SearchQuery len00008 256 15 15 768 0.9999005502401808 0.0023620740898110237 equal +Q8_0 SearchQuery len00008 128 15 15 768 0.9999318387771912 0.0027679763473324546 equal +Q8_0 SearchQuery len00128 768 143 143 768 0.9999376974695419 0.0014073896208196962 equal +Q8_0 SearchQuery len00128 512 143 143 768 0.9999364727603882 0.001618243751416705 equal +Q8_0 SearchQuery len00128 256 143 143 768 0.9999460124271976 0.001990053194536927 equal +Q8_0 SearchQuery len00128 128 143 143 768 0.999959997261044 0.001914805614829096 equal +Q8_0 SearchQuery len01024 768 1148 1148 768 0.9999521290016042 0.00135487193987021 equal +Q8_0 SearchQuery len01024 512 1148 1148 768 0.9999545601256676 0.0015447208814409544 equal +Q8_0 SearchQuery len01024 256 1148 1148 768 0.9999605952215678 0.0019345120117140423 equal +Q8_0 SearchQuery len01024 128 1148 1148 768 0.9999712711191562 0.002254178356785222 equal +Q8_0 SearchQuery len02000 768 2406 2406 768 0.9999504539863175 0.0012751038595257731 equal +Q8_0 SearchQuery len02000 512 2406 2406 768 0.999951422095172 0.0014533996175640773 equal +Q8_0 SearchQuery len02000 256 2406 2406 768 0.9999587881833429 0.0018050992598480052 equal +Q8_0 SearchQuery len02000 128 2406 2406 768 0.999975758668811 0.0019274271211755512 equal +Q8_0 SearchQuery len08000 768 11402 11402 768 0.9999763347624723 0.0008151603651409337 equal +Q8_0 SearchQuery len08000 512 11402 11402 768 0.9999779030185171 0.000925138292496526 equal +Q8_0 SearchQuery len08000 256 11402 11402 768 0.9999805970535006 0.0010654280951503392 equal +Q8_0 SearchQuery len08000 128 11402 11402 768 0.999987550840208 0.0011185914396747987 equal +Q8_0 QuestionAnswering len00008 768 15 15 768 0.9998438511277902 0.0025771266673388915 equal +Q8_0 QuestionAnswering len00008 512 15 15 768 0.9998399262143514 0.002974322901864425 equal +Q8_0 QuestionAnswering len00008 256 15 15 768 0.9998592622107655 0.003127505832307892 equal +Q8_0 QuestionAnswering len00008 128 15 15 768 0.9999215009543185 0.0024461929060788437 equal +Q8_0 QuestionAnswering len00128 768 143 143 768 0.9999001260425213 0.002046054748369033 equal +Q8_0 QuestionAnswering len00128 512 143 143 768 0.9998983252978552 0.002354110478564911 equal +Q8_0 QuestionAnswering len00128 256 143 143 768 0.9999107889265454 0.0029175952777585525 equal +Q8_0 QuestionAnswering len00128 128 143 143 768 0.9999372640136872 0.0034694182774006165 equal +Q8_0 QuestionAnswering len01024 768 1148 1148 768 0.9999367250079636 0.0013533958505988737 equal +Q8_0 QuestionAnswering len01024 512 1148 1148 768 0.9999376678732 0.0015436945966675608 equal +Q8_0 QuestionAnswering len01024 256 1148 1148 768 0.999944899656683 0.001922518502798112 equal +Q8_0 QuestionAnswering len01024 128 1148 1148 768 0.9999622881305669 0.002299413692062661 equal +Q8_0 QuestionAnswering len02000 768 2406 2406 768 0.9999372728628382 0.0012077746101582095 equal +Q8_0 QuestionAnswering len02000 512 2406 2406 768 0.9999385175511826 0.0013792460011533582 equal +Q8_0 QuestionAnswering len02000 256 2406 2406 768 0.9999462531145678 0.001725118140606895 equal +Q8_0 QuestionAnswering len02000 128 2406 2406 768 0.9999663647169773 0.0018408435360768652 equal +Q8_0 QuestionAnswering len08000 768 11402 11402 768 0.9999814354704492 0.0007919591653615249 equal +Q8_0 QuestionAnswering len08000 512 11402 11402 768 0.9999828346106915 0.0009068212944301629 equal +Q8_0 QuestionAnswering len08000 256 11402 11402 768 0.9999844360302517 0.001155706460115133 equal +Q8_0 QuestionAnswering len08000 128 11402 11402 768 0.999988596305254 0.001332962284986236 equal +Q8_0 FactChecking len00008 768 15 15 768 0.999771572001089 0.002767294614696105 equal +Q8_0 FactChecking len00008 512 15 15 768 0.9997801657354453 0.003185220393176178 equal +Q8_0 FactChecking len00008 256 15 15 768 0.9997933568833068 0.003985184506109244 equal +Q8_0 FactChecking len00008 128 15 15 768 0.999829954364001 0.004593102918430823 equal +Q8_0 FactChecking len00128 768 143 143 768 0.9998380099272726 0.0022611914063551714 equal +Q8_0 FactChecking len00128 512 143 143 768 0.9998365372950426 0.0026003538725082307 equal +Q8_0 FactChecking len00128 256 143 143 768 0.999855533774147 0.003246625705308242 equal +Q8_0 FactChecking len00128 128 143 143 768 0.999874426938539 0.003934290025130131 equal +Q8_0 FactChecking len01024 768 1148 1148 768 0.9999302196472446 0.001729806835546549 equal +Q8_0 FactChecking len01024 512 1148 1148 768 0.9999338026815069 0.0019611820157438287 equal +Q8_0 FactChecking len01024 256 1148 1148 768 0.9999404206649477 0.0024724123739715 equal +Q8_0 FactChecking len01024 128 1148 1148 768 0.9999553332890734 0.0029688829819558282 equal +Q8_0 FactChecking len02000 768 2406 2406 768 0.9999178662980966 0.0016009172610424727 equal +Q8_0 FactChecking len02000 512 2406 2406 768 0.9999210630015783 0.0018282576751105506 equal +Q8_0 FactChecking len02000 256 2406 2406 768 0.9999330224038347 0.002265367445914171 equal +Q8_0 FactChecking len02000 128 2406 2406 768 0.9999542638892721 0.0026594074838590387 equal +Q8_0 FactChecking len08000 768 11402 11402 768 0.9999786643633027 0.0007157746775813155 equal +Q8_0 FactChecking len08000 512 11402 11402 768 0.9999800996212118 0.0008161927785689917 equal +Q8_0 FactChecking len08000 256 11402 11402 768 0.9999798065029101 0.0010318192906221251 equal +Q8_0 FactChecking len08000 128 11402 11402 768 0.9999875305049155 0.0011850501239846975 equal +Q8_0 CodeRetrieval len00008 768 15 15 768 0.9998974850140855 0.0016188821990971401 equal +Q8_0 CodeRetrieval len00008 512 15 15 768 0.9999015968777472 0.001861388055640744 equal +Q8_0 CodeRetrieval len00008 256 15 15 768 0.9999086230479983 0.0023217160949426316 equal +Q8_0 CodeRetrieval len00008 128 15 15 768 0.9999418865106817 0.0024269504661902017 equal +Q8_0 CodeRetrieval len00128 768 143 143 768 0.999876310543174 0.0024768548587970965 equal +Q8_0 CodeRetrieval len00128 512 143 143 768 0.9998817513535521 0.0028348369274891344 equal +Q8_0 CodeRetrieval len00128 256 143 143 768 0.9998962363906535 0.0035137407345132154 equal +Q8_0 CodeRetrieval len00128 128 143 143 768 0.9999274444359912 0.0040013371996150265 equal +Q8_0 CodeRetrieval len01024 768 1148 1148 768 0.9999301979932511 0.001532630546774806 equal +Q8_0 CodeRetrieval len01024 512 1148 1148 768 0.99993089789614 0.0017582289487782093 equal +Q8_0 CodeRetrieval len01024 256 1148 1148 768 0.9999362907602852 0.0021832992749088592 equal +Q8_0 CodeRetrieval len01024 128 1148 1148 768 0.9999394821145019 0.0025354211601641974 equal +Q8_0 CodeRetrieval len02000 768 2406 2406 768 0.9999279451884636 0.0018779139409756535 equal +Q8_0 CodeRetrieval len02000 512 2406 2406 768 0.9999300570505066 0.0021266559237488464 equal +Q8_0 CodeRetrieval len02000 256 2406 2406 768 0.9999343344956032 0.002661305575936204 equal +Q8_0 CodeRetrieval len02000 128 2406 2406 768 0.9999544052457937 0.0029948566465852755 equal +Q8_0 CodeRetrieval len08000 768 11402 11402 768 0.9999801575764794 0.0009034583610264962 equal +Q8_0 CodeRetrieval len08000 512 11402 11402 768 0.9999807030704164 0.0010158228239763867 equal +Q8_0 CodeRetrieval len08000 256 11402 11402 768 0.9999831921855136 0.001275441875403488 equal +Q8_0 CodeRetrieval len08000 128 11402 11402 768 0.9999887957425521 0.0014945638273858586 equal +Q8_0 Classification len00008 768 14 14 768 0.9998447639144105 0.001849085636981095 equal +Q8_0 Classification len00008 512 14 14 768 0.9998521620922098 0.002127584648575348 equal +Q8_0 Classification len00008 256 14 14 768 0.999865658092123 0.0026709966842615365 equal +Q8_0 Classification len00008 128 14 14 768 0.9999076186594112 0.003080476622801776 equal +Q8_0 Classification len00128 768 142 142 768 0.9999102746226618 0.002042380328924748 equal +Q8_0 Classification len00128 512 142 142 768 0.9999088071868435 0.00233739169397945 equal +Q8_0 Classification len00128 256 142 142 768 0.9999179828413888 0.002900973733915515 equal +Q8_0 Classification len00128 128 142 142 768 0.9999482803687434 0.002895624393271113 equal +Q8_0 Classification len01024 768 1147 1147 768 0.9999410150365817 0.0015758822748282234 equal +Q8_0 Classification len01024 512 1147 1147 768 0.9999430046893378 0.001797295588661811 equal +Q8_0 Classification len01024 256 1147 1147 768 0.9999471072831391 0.0022429273012288115 equal +Q8_0 Classification len01024 128 1147 1147 768 0.9999633719177232 0.0017988400990575915 equal +Q8_0 Classification len02000 768 2405 2405 768 0.9999385829877114 0.0015163372813357077 equal +Q8_0 Classification len02000 512 2405 2405 768 0.999938532584455 0.0017281080365984525 equal +Q8_0 Classification len02000 256 2405 2405 768 0.9999424903566378 0.002154554329012886 equal +Q8_0 Classification len02000 128 2405 2405 768 0.9999621940514435 0.002439808061210365 equal +Q8_0 Classification len08000 768 11401 11401 768 0.9999810160917753 0.0006355914810390294 equal +Q8_0 Classification len08000 512 11401 11401 768 0.9999822934761379 0.0007271585640462458 equal +Q8_0 Classification len08000 256 11401 11401 768 0.9999838974552325 0.0009034365682651785 equal +Q8_0 Classification len08000 128 11401 11401 768 0.999989254285509 0.0009844422256690133 equal +Q8_0 Clustering len00008 768 14 14 768 0.9999440946874233 0.0013531884004988302 equal +Q8_0 Clustering len00008 512 14 14 768 0.9999448686865414 0.0015532390622006644 equal +Q8_0 Clustering len00008 256 14 14 768 0.9999525599241732 0.0019327437890423824 equal +Q8_0 Clustering len00008 128 14 14 768 0.9999661810452689 0.002169401648567045 equal +Q8_0 Clustering len00128 768 142 142 768 0.9999520868587786 0.001236383482235445 equal +Q8_0 Clustering len00128 512 142 142 768 0.99995355197287 0.0014098891202129205 equal +Q8_0 Clustering len00128 256 142 142 768 0.9999600783500722 0.0014106438528081328 equal +Q8_0 Clustering len00128 128 142 142 768 0.9999723905461653 0.0015182540607995426 equal +Q8_0 Clustering len01024 768 1147 1147 768 0.9999467757823746 0.0012850445527417178 equal +Q8_0 Clustering len01024 512 1147 1147 768 0.9999492086056108 0.0014676006352092718 equal +Q8_0 Clustering len01024 256 1147 1147 768 0.9999559539427666 0.0018049726715963811 equal +Q8_0 Clustering len01024 128 1147 1147 768 0.9999734747771188 0.00204753910252653 equal +Q8_0 Clustering len02000 768 2405 2405 768 0.9999507836859481 0.0013325608472114477 equal +Q8_0 Clustering len02000 512 2405 2405 768 0.9999509115106964 0.0015192318314969757 equal +Q8_0 Clustering len02000 256 2405 2405 768 0.9999592518102846 0.0018697230314552128 equal +Q8_0 Clustering len02000 128 2405 2405 768 0.9999702620036612 0.002108298535782399 equal +Q8_0 Clustering len08000 768 11401 11401 768 0.9999812817678428 0.0007637010589795594 equal +Q8_0 Clustering len08000 512 11401 11401 768 0.9999809599911891 0.0008681854562183448 equal +Q8_0 Clustering len08000 256 11401 11401 768 0.999982953027126 0.0010835418997558674 equal +Q8_0 Clustering len08000 128 11401 11401 768 0.9999888808417295 0.0012197047662854685 equal +Q8_0 SentenceSimilarity len00008 768 15 15 768 0.9999307295218882 0.0014342126563065896 equal +Q8_0 SentenceSimilarity len00008 512 15 15 768 0.9999292290444965 0.0016466814038237007 equal +Q8_0 SentenceSimilarity len00008 256 15 15 768 0.9999360543879913 0.002027340354513671 equal +Q8_0 SentenceSimilarity len00008 128 15 15 768 0.9999567527567291 0.0023120196701151563 equal +Q8_0 SentenceSimilarity len00128 768 143 143 768 0.9999555959915759 0.0011723122004770083 equal +Q8_0 SentenceSimilarity len00128 512 143 143 768 0.9999560604152858 0.001350883946747928 equal +Q8_0 SentenceSimilarity len00128 256 143 143 768 0.9999606980973411 0.0016931261557193433 equal +Q8_0 SentenceSimilarity len00128 128 143 143 768 0.9999674490504353 0.0019714655763946893 equal +Q8_0 SentenceSimilarity len01024 768 1148 1148 768 0.9999608646854807 0.0010724618621880478 equal +Q8_0 SentenceSimilarity len01024 512 1148 1148 768 0.9999603332398513 0.0012239170082526925 equal +Q8_0 SentenceSimilarity len01024 256 1148 1148 768 0.9999635731571741 0.0015279957947571515 equal +Q8_0 SentenceSimilarity len01024 128 1148 1148 768 0.9999768271250173 0.001778453191059133 equal +Q8_0 SentenceSimilarity len02000 768 2406 2406 768 0.9999636663385001 0.0012735083263287136 equal +Q8_0 SentenceSimilarity len02000 512 2406 2406 768 0.9999639244160158 0.0014531076509568114 equal +Q8_0 SentenceSimilarity len02000 256 2406 2406 768 0.999968784223929 0.0018152268192904025 equal +Q8_0 SentenceSimilarity len02000 128 2406 2406 768 0.9999832627327716 0.0021005701010428646 equal +Q8_0 SentenceSimilarity len08000 768 11402 11402 768 0.9999840773272262 0.0009186763394654473 equal +Q8_0 SentenceSimilarity len08000 512 11402 11402 768 0.9999847337834623 0.0010476827439435382 equal +Q8_0 SentenceSimilarity len08000 256 11402 11402 768 0.9999849218047494 0.0013199319648051593 equal +Q8_0 SentenceSimilarity len08000 128 11402 11402 768 0.9999907018540616 0.0009886836127106363 equal +UD-Q6_K_XL SearchQuery len00008 768 15 15 768 0.9997207356139964 0.002787769233460389 equal +UD-Q6_K_XL SearchQuery len00008 512 15 15 768 0.9997214466910128 0.003228854942093816 equal +UD-Q6_K_XL SearchQuery len00008 256 15 15 768 0.9997605894860518 0.004044654235561626 equal +UD-Q6_K_XL SearchQuery len00008 128 15 15 768 0.9998166238796017 0.004418817257559313 equal +UD-Q6_K_XL SearchQuery len00128 768 143 143 768 0.9997745010005162 0.0023557539505232655 equal +UD-Q6_K_XL SearchQuery len00128 512 143 143 768 0.999787577867026 0.0027186926731328533 equal +UD-Q6_K_XL SearchQuery len00128 256 143 143 768 0.99983507726204 0.0033378761560591786 equal +UD-Q6_K_XL SearchQuery len00128 128 143 143 768 0.9998781382498241 0.0032424909683992924 equal +UD-Q6_K_XL SearchQuery len01024 768 1148 1148 768 0.9998210743077561 0.0022915355156791425 equal +UD-Q6_K_XL SearchQuery len01024 512 1148 1148 768 0.9998365910556353 0.0026067846918838705 equal +UD-Q6_K_XL SearchQuery len01024 256 1148 1148 768 0.9998540712920723 0.003233286288638254 equal +UD-Q6_K_XL SearchQuery len01024 128 1148 1148 768 0.9999127839823586 0.0033897381914216804 equal +UD-Q6_K_XL SearchQuery len02000 768 2406 2406 768 0.9998651802786414 0.001791215905074332 equal +UD-Q6_K_XL SearchQuery len02000 512 2406 2406 768 0.9998670889026678 0.002046411673799981 equal +UD-Q6_K_XL SearchQuery len02000 256 2406 2406 768 0.9998786901512574 0.002516325618178772 equal +UD-Q6_K_XL SearchQuery len02000 128 2406 2406 768 0.9999162277114545 0.0029402683985562186 equal +UD-Q6_K_XL SearchQuery len08000 768 11402 11402 768 0.9999314875172137 0.0018334413666827101 equal +UD-Q6_K_XL SearchQuery len08000 512 11402 11402 768 0.999935383284602 0.002086929142572448 equal +UD-Q6_K_XL SearchQuery len08000 256 11402 11402 768 0.9999460549198195 0.0026165964766587183 equal +UD-Q6_K_XL SearchQuery len08000 128 11402 11402 768 0.9999588905784129 0.0029890380957901877 equal +UD-Q6_K_XL QuestionAnswering len00008 768 15 15 768 0.9995448287268626 0.0033498406468925636 equal +UD-Q6_K_XL QuestionAnswering len00008 512 15 15 768 0.9995669357551905 0.003869262060517185 equal +UD-Q6_K_XL QuestionAnswering len00008 256 15 15 768 0.9996141626103725 0.00463315416391398 equal +UD-Q6_K_XL QuestionAnswering len00008 128 15 15 768 0.9997386260345194 0.005407185542076226 equal +UD-Q6_K_XL QuestionAnswering len00128 768 143 143 768 0.9995924521138713 0.004540893881034 equal +UD-Q6_K_XL QuestionAnswering len00128 512 143 143 768 0.9996071845774332 0.005167789077763191 equal +UD-Q6_K_XL QuestionAnswering len00128 256 143 143 768 0.9996668217884559 0.006300209684300076 equal +UD-Q6_K_XL QuestionAnswering len00128 128 143 143 768 0.9997344801824145 0.007379937849666715 equal +UD-Q6_K_XL QuestionAnswering len01024 768 1148 1148 768 0.9997367223206708 0.0029483004978656563 equal +UD-Q6_K_XL QuestionAnswering len01024 512 1148 1148 768 0.9997321112420292 0.0033632230702668503 equal +UD-Q6_K_XL QuestionAnswering len01024 256 1148 1148 768 0.9997659510841962 0.0036474796503784546 equal +UD-Q6_K_XL QuestionAnswering len01024 128 1148 1148 768 0.9998462688571581 0.0036738794579616457 equal +UD-Q6_K_XL QuestionAnswering len02000 768 2406 2406 768 0.9998383713773027 0.002408541005034348 equal +UD-Q6_K_XL QuestionAnswering len02000 512 2406 2406 768 0.9998413966563343 0.002821803599073913 equal +UD-Q6_K_XL QuestionAnswering len02000 256 2406 2406 768 0.999860083536929 0.003265162575695646 equal +UD-Q6_K_XL QuestionAnswering len02000 128 2406 2406 768 0.9999042242563144 0.0035014022107354714 equal +UD-Q6_K_XL QuestionAnswering len08000 768 11402 11402 768 0.9999387328900274 0.0012874620359039947 equal +UD-Q6_K_XL QuestionAnswering len08000 512 11402 11402 768 0.9999431260118166 0.0014649318121176987 equal +UD-Q6_K_XL QuestionAnswering len08000 256 11402 11402 768 0.9999468180667301 0.001842693060651053 equal +UD-Q6_K_XL QuestionAnswering len08000 128 11402 11402 768 0.9999633411520047 0.0018706251301077248 equal +UD-Q6_K_XL FactChecking len00008 768 15 15 768 0.9992351688696751 0.004537692518713266 equal +UD-Q6_K_XL FactChecking len00008 512 15 15 768 0.9992453374210959 0.005233316857375613 equal +UD-Q6_K_XL FactChecking len00008 256 15 15 768 0.9993490133611229 0.005376830754675665 equal +UD-Q6_K_XL FactChecking len00008 128 15 15 768 0.9995339498585993 0.006229322013919868 equal +UD-Q6_K_XL FactChecking len00128 768 143 143 768 0.9995032179039444 0.0038838222258229804 equal +UD-Q6_K_XL FactChecking len00128 512 143 143 768 0.9995129043390175 0.004463276799187881 equal +UD-Q6_K_XL FactChecking len00128 256 143 143 768 0.999545408431339 0.005543994652110005 equal +UD-Q6_K_XL FactChecking len00128 128 143 143 768 0.9996943781789658 0.005640007321715372 equal +UD-Q6_K_XL FactChecking len01024 768 1148 1148 768 0.9997943523983768 0.002861467471072028 equal +UD-Q6_K_XL FactChecking len01024 512 1148 1148 768 0.9998043452735014 0.003244618890720464 equal +UD-Q6_K_XL FactChecking len01024 256 1148 1148 768 0.9998362349185899 0.004043381621151761 equal +UD-Q6_K_XL FactChecking len01024 128 1148 1148 768 0.999873739223654 0.004778245456313064 equal +UD-Q6_K_XL FactChecking len02000 768 2406 2406 768 0.9997591018980828 0.0022304798104417073 equal +UD-Q6_K_XL FactChecking len02000 512 2406 2406 768 0.9997718031907138 0.0025366442832099254 equal +UD-Q6_K_XL FactChecking len02000 256 2406 2406 768 0.9997996281104525 0.0030810795811531642 equal +UD-Q6_K_XL FactChecking len02000 128 2406 2406 768 0.9998536761641205 0.0035762650383732927 equal +UD-Q6_K_XL FactChecking len08000 768 11402 11402 768 0.9999349397157735 0.001473882256907262 equal +UD-Q6_K_XL FactChecking len08000 512 11402 11402 768 0.9999388373424687 0.0016824219028643665 equal +UD-Q6_K_XL FactChecking len08000 256 11402 11402 768 0.999949193380923 0.0021087939662476907 equal +UD-Q6_K_XL FactChecking len08000 128 11402 11402 768 0.9999636054994925 0.0022691457593997516 equal +UD-Q6_K_XL CodeRetrieval len00008 768 15 15 768 0.9996287051288723 0.0029952088176569244 equal +UD-Q6_K_XL CodeRetrieval len00008 512 15 15 768 0.9996372205366503 0.0034238130556112495 equal +UD-Q6_K_XL CodeRetrieval len00008 256 15 15 768 0.99970642956024 0.004273505778993283 equal +UD-Q6_K_XL CodeRetrieval len00008 128 15 15 768 0.9998084869249778 0.004889440995283975 equal +UD-Q6_K_XL CodeRetrieval len00128 768 143 143 768 0.9996745376063367 0.0032242051743278863 equal +UD-Q6_K_XL CodeRetrieval len00128 512 143 143 768 0.9996758677198346 0.0036939430115476082 equal +UD-Q6_K_XL CodeRetrieval len00128 256 143 143 768 0.9997007914990845 0.004558185903000431 equal +UD-Q6_K_XL CodeRetrieval len00128 128 143 143 768 0.9998082191345851 0.004447075859787798 equal +UD-Q6_K_XL CodeRetrieval len01024 768 1148 1148 768 0.9998180391076614 0.002044344888446077 equal +UD-Q6_K_XL CodeRetrieval len01024 512 1148 1148 768 0.9998260478511051 0.0023141258370708156 equal +UD-Q6_K_XL CodeRetrieval len01024 256 1148 1148 768 0.9998398995079978 0.002937831985184561 equal +UD-Q6_K_XL CodeRetrieval len01024 128 1148 1148 768 0.9998982288846341 0.003121800962193555 equal +UD-Q6_K_XL CodeRetrieval len02000 768 2406 2406 768 0.9998374017993485 0.0022915841116717957 equal +UD-Q6_K_XL CodeRetrieval len02000 512 2406 2406 768 0.9998425130441166 0.0026115433379943283 equal +UD-Q6_K_XL CodeRetrieval len02000 256 2406 2406 768 0.9998612364202325 0.0032725977833159827 equal +UD-Q6_K_XL CodeRetrieval len02000 128 2406 2406 768 0.9999165133313738 0.003218626753715148 equal +UD-Q6_K_XL CodeRetrieval len08000 768 11402 11402 768 0.9999412773500097 0.0014240462244696633 equal +UD-Q6_K_XL CodeRetrieval len08000 512 11402 11402 768 0.9999474801686189 0.001619068019077751 equal +UD-Q6_K_XL CodeRetrieval len08000 256 11402 11402 768 0.9999526642659411 0.0020393402296228977 equal +UD-Q6_K_XL CodeRetrieval len08000 128 11402 11402 768 0.999965467957939 0.0023133239186707844 equal +UD-Q6_K_XL Classification len00008 768 14 14 768 0.9994092510119001 0.004090521134456036 equal +UD-Q6_K_XL Classification len00008 512 14 14 768 0.999435670631921 0.004734582291633024 equal +UD-Q6_K_XL Classification len00008 256 14 14 768 0.999495253120944 0.005960581275385013 equal +UD-Q6_K_XL Classification len00008 128 14 14 768 0.9996353732429087 0.006756951884454229 equal +UD-Q6_K_XL Classification len00128 768 142 142 768 0.9997918583137729 0.003248230552299263 equal +UD-Q6_K_XL Classification len00128 512 142 142 768 0.9998018920334933 0.0037252867301281432 equal +UD-Q6_K_XL Classification len00128 256 142 142 768 0.9998304811224836 0.004621342664002742 equal +UD-Q6_K_XL Classification len00128 128 142 142 768 0.9999096858467379 0.003316264033780403 equal +UD-Q6_K_XL Classification len01024 768 1147 1147 768 0.999840223416118 0.002200186428937368 equal +UD-Q6_K_XL Classification len01024 512 1147 1147 768 0.9998514088203052 0.002325175358992758 equal +UD-Q6_K_XL Classification len01024 256 1147 1147 768 0.9998766289589015 0.0028861449887207374 equal +UD-Q6_K_XL Classification len01024 128 1147 1147 768 0.9999121350859717 0.003314313125291242 equal +UD-Q6_K_XL Classification len02000 768 2405 2405 768 0.999808034918751 0.0025299661261938 equal +UD-Q6_K_XL Classification len02000 512 2405 2405 768 0.9998255208094473 0.0028916137077835205 equal +UD-Q6_K_XL Classification len02000 256 2405 2405 768 0.9998431837020594 0.0035952949977725787 equal +UD-Q6_K_XL Classification len02000 128 2405 2405 768 0.9999033097275494 0.003264167986118352 equal +UD-Q6_K_XL Classification len08000 768 11401 11401 768 0.9999459163966058 0.0012141148489428419 equal +UD-Q6_K_XL Classification len08000 512 11401 11401 768 0.9999491532926089 0.001379133046359015 equal +UD-Q6_K_XL Classification len08000 256 11401 11401 768 0.9999575110390299 0.001738220337680143 equal +UD-Q6_K_XL Classification len08000 128 11401 11401 768 0.9999692022936727 0.0019797643517409647 equal +UD-Q6_K_XL Clustering len00008 768 14 14 768 0.9998225924806299 0.00211593632831371 equal +UD-Q6_K_XL Clustering len00008 512 14 14 768 0.9998283021730706 0.0024321793812816846 equal +UD-Q6_K_XL Clustering len00008 256 14 14 768 0.9998472082185437 0.0028321488711130494 equal +UD-Q6_K_XL Clustering len00008 128 14 14 768 0.9998974064516787 0.003155356451124608 equal +UD-Q6_K_XL Clustering len00128 768 142 142 768 0.999887606474686 0.0018523129119407539 equal +UD-Q6_K_XL Clustering len00128 512 142 142 768 0.999893248695449 0.0021221412892701713 equal +UD-Q6_K_XL Clustering len00128 256 142 142 768 0.9999070658069075 0.002592515433921541 equal +UD-Q6_K_XL Clustering len00128 128 142 142 768 0.9999329623535023 0.003010131940315755 equal +UD-Q6_K_XL Clustering len01024 768 1147 1147 768 0.9998410296970964 0.002588357628788572 equal +UD-Q6_K_XL Clustering len01024 512 1147 1147 768 0.9998455390542983 0.0029544210492072377 equal +UD-Q6_K_XL Clustering len01024 256 1147 1147 768 0.999861615106384 0.003639386154872589 equal +UD-Q6_K_XL Clustering len01024 128 1147 1147 768 0.9998889001210822 0.004123955781560057 equal +UD-Q6_K_XL Clustering len02000 768 2405 2405 768 0.9998535021056207 0.002230671924389891 equal +UD-Q6_K_XL Clustering len02000 512 2405 2405 768 0.9998526155442697 0.002550511908629649 equal +UD-Q6_K_XL Clustering len02000 256 2405 2405 768 0.9998659945884211 0.003115515691233639 equal +UD-Q6_K_XL Clustering len02000 128 2405 2405 768 0.9999139762111607 0.003507424682487055 equal +UD-Q6_K_XL Clustering len08000 768 11401 11401 768 0.9999343810812763 0.0015529092367523346 equal +UD-Q6_K_XL Clustering len08000 512 11401 11401 768 0.9999364636455766 0.0017665179621619165 equal +UD-Q6_K_XL Clustering len08000 256 11401 11401 768 0.999941015075334 0.0021841897805323 equal +UD-Q6_K_XL Clustering len08000 128 11401 11401 768 0.999953167051607 0.0021896550685435506 equal +UD-Q6_K_XL SentenceSimilarity len00008 768 15 15 768 0.9998465404096275 0.0023997295083673137 equal +UD-Q6_K_XL SentenceSimilarity len00008 512 15 15 768 0.9998489158218787 0.0027619120537467085 equal +UD-Q6_K_XL SentenceSimilarity len00008 256 15 15 768 0.9998651682178941 0.003463961063370635 equal +UD-Q6_K_XL SentenceSimilarity len00008 128 15 15 768 0.9999229175379721 0.003964956855968649 equal +UD-Q6_K_XL SentenceSimilarity len00128 768 143 143 768 0.9999111245125554 0.001791104138284834 equal +UD-Q6_K_XL SentenceSimilarity len00128 512 143 143 768 0.9999128524625731 0.00206901162212704 equal +UD-Q6_K_XL SentenceSimilarity len00128 256 143 143 768 0.9999259987594403 0.002564427955352902 equal +UD-Q6_K_XL SentenceSimilarity len00128 128 143 143 768 0.9999489700464691 0.003017981995676544 equal +UD-Q6_K_XL SentenceSimilarity len01024 768 1148 1148 768 0.999861919531487 0.0023489848146684145 equal +UD-Q6_K_XL SentenceSimilarity len01024 512 1148 1148 768 0.99986047599524 0.002678886977785063 equal +UD-Q6_K_XL SentenceSimilarity len01024 256 1148 1148 768 0.9998828466528691 0.003348473180976301 equal +UD-Q6_K_XL SentenceSimilarity len01024 128 1148 1148 768 0.9999088695682964 0.003875237307823484 equal +UD-Q6_K_XL SentenceSimilarity len02000 768 2406 2406 768 0.9998952101707945 0.0016030022384420466 equal +UD-Q6_K_XL SentenceSimilarity len02000 512 2406 2406 768 0.9998965272640262 0.0018325567310263496 equal +UD-Q6_K_XL SentenceSimilarity len02000 256 2406 2406 768 0.9999020302380621 0.0022740304847732283 equal +UD-Q6_K_XL SentenceSimilarity len02000 128 2406 2406 768 0.999933863447793 0.0025977438816396393 equal +UD-Q6_K_XL SentenceSimilarity len08000 768 11402 11402 768 0.9999451795773032 0.0011736305492946845 equal +UD-Q6_K_XL SentenceSimilarity len08000 512 11402 11402 768 0.999947868009996 0.0013386036275382755 equal +UD-Q6_K_XL SentenceSimilarity len08000 256 11402 11402 768 0.9999545259449375 0.0016668374528537137 equal +UD-Q6_K_XL SentenceSimilarity len08000 128 11402 11402 768 0.9999647691574645 0.0017769619505630958 equal +UD-Q5_K_XL SearchQuery len00008 768 15 15 768 0.9996992250892776 0.0031002453702016625 equal +UD-Q5_K_XL SearchQuery len00008 512 15 15 768 0.9997035924150857 0.003580466929642097 equal +UD-Q5_K_XL SearchQuery len00008 256 15 15 768 0.9997091831535332 0.004433629979332623 equal +UD-Q5_K_XL SearchQuery len00008 128 15 15 768 0.9997609689222464 0.005249707777824492 equal +UD-Q5_K_XL SearchQuery len00128 768 143 143 768 0.9997703665725232 0.0032287313785729892 equal +UD-Q5_K_XL SearchQuery len00128 512 143 143 768 0.9997730661201599 0.003717464361200875 equal +UD-Q5_K_XL SearchQuery len00128 256 143 143 768 0.9998032913413427 0.004577053062060438 equal +UD-Q5_K_XL SearchQuery len00128 128 143 143 768 0.9998466558820266 0.005546893830055893 equal +UD-Q5_K_XL SearchQuery len01024 768 1148 1148 768 0.9998244980074129 0.0020005563078626595 equal +UD-Q5_K_XL SearchQuery len01024 512 1148 1148 768 0.9998291694959878 0.0022714006445348246 equal +UD-Q5_K_XL SearchQuery len01024 256 1148 1148 768 0.9998421507150254 0.002844402326787232 equal +UD-Q5_K_XL SearchQuery len01024 128 1148 1148 768 0.9998918700881299 0.003484371720325216 equal +UD-Q5_K_XL SearchQuery len02000 768 2406 2406 768 0.9998123503718627 0.0026830652672665614 equal +UD-Q5_K_XL SearchQuery len02000 512 2406 2406 768 0.9998172683562663 0.003067517008463014 equal +UD-Q5_K_XL SearchQuery len02000 256 2406 2406 768 0.9998494996641695 0.003861787354621421 equal +UD-Q5_K_XL SearchQuery len02000 128 2406 2406 768 0.999916184537642 0.0032373353780702774 equal +UD-Q5_K_XL SearchQuery len08000 768 11402 11402 768 0.9999284808328035 0.0011758529916546454 equal +UD-Q5_K_XL SearchQuery len08000 512 11402 11402 768 0.9999306589166788 0.0013052766561643787 equal +UD-Q5_K_XL SearchQuery len08000 256 11402 11402 768 0.9999405350250369 0.001609456670113675 equal +UD-Q5_K_XL SearchQuery len08000 128 11402 11402 768 0.9999651422084175 0.0017973987232358171 equal +UD-Q5_K_XL QuestionAnswering len00008 768 15 15 768 0.9996858005069332 0.002748856841283634 equal +UD-Q5_K_XL QuestionAnswering len00008 512 15 15 768 0.9997015393498545 0.003162600119000644 equal +UD-Q5_K_XL QuestionAnswering len00008 256 15 15 768 0.9997521092578172 0.003956795810918831 equal +UD-Q5_K_XL QuestionAnswering len00008 128 15 15 768 0.999805659521373 0.004602518538430786 equal +UD-Q5_K_XL QuestionAnswering len00128 768 143 143 768 0.9995837780977164 0.003522995533924461 equal +UD-Q5_K_XL QuestionAnswering len00128 512 143 143 768 0.9996019696924267 0.0040601526614621845 equal +UD-Q5_K_XL QuestionAnswering len00128 256 143 143 768 0.9996327084222081 0.005021005062664674 equal +UD-Q5_K_XL QuestionAnswering len00128 128 143 143 768 0.9997398675641928 0.006016141924514745 equal +UD-Q5_K_XL QuestionAnswering len01024 768 1148 1148 768 0.9997871524669331 0.0022936043294530875 equal +UD-Q5_K_XL QuestionAnswering len01024 512 1148 1148 768 0.9998009496216229 0.0026139205694482442 equal +UD-Q5_K_XL QuestionAnswering len01024 256 1148 1148 768 0.999834076381186 0.003232292921499229 equal +UD-Q5_K_XL QuestionAnswering len01024 128 1148 1148 768 0.9999038820129673 0.003114614670468259 equal +UD-Q5_K_XL QuestionAnswering len02000 768 2406 2406 768 0.9997870225376966 0.0024861598801937113 equal +UD-Q5_K_XL QuestionAnswering len02000 512 2406 2406 768 0.9997882993318232 0.002841792755413441 equal +UD-Q5_K_XL QuestionAnswering len02000 256 2406 2406 768 0.999798079608071 0.0035589027867174803 equal +UD-Q5_K_XL QuestionAnswering len02000 128 2406 2406 768 0.9998721578487906 0.003587496963059546 equal +UD-Q5_K_XL QuestionAnswering len08000 768 11402 11402 768 0.9999104764712927 0.0016113568778539852 equal +UD-Q5_K_XL QuestionAnswering len08000 512 11402 11402 768 0.9999136800566639 0.0016998250412902163 equal +UD-Q5_K_XL QuestionAnswering len08000 256 11402 11402 768 0.9999253310888054 0.0020559021007676946 equal +UD-Q5_K_XL QuestionAnswering len08000 128 11402 11402 768 0.9999573756920481 0.002231229181645717 equal +UD-Q5_K_XL FactChecking len00008 768 15 15 768 0.9988871869785967 0.00504115897751186 equal +UD-Q5_K_XL FactChecking len00008 512 15 15 768 0.9988614824128205 0.00582218924274331 equal +UD-Q5_K_XL FactChecking len00008 256 15 15 768 0.9989917773424518 0.007278915460931457 equal +UD-Q5_K_XL FactChecking len00008 128 15 15 768 0.9993531980746199 0.008147525933012354 equal +UD-Q5_K_XL FactChecking len00128 768 143 143 768 0.9995708801754865 0.0037909860964932263 equal +UD-Q5_K_XL FactChecking len00128 512 143 143 768 0.9995946625482669 0.00435747535762121 equal +UD-Q5_K_XL FactChecking len00128 256 143 143 768 0.9996722654085599 0.005398731297057979 equal +UD-Q5_K_XL FactChecking len00128 128 143 143 768 0.9997755617947993 0.004489761527146731 equal +UD-Q5_K_XL FactChecking len01024 768 1148 1148 768 0.9997789407038488 0.0028331158329277274 equal +UD-Q5_K_XL FactChecking len01024 512 1148 1148 768 0.9998021698298007 0.0032382488231976377 equal +UD-Q5_K_XL FactChecking len01024 256 1148 1148 768 0.9998170727350764 0.0040403510780816585 equal +UD-Q5_K_XL FactChecking len01024 128 1148 1148 768 0.9998923080107173 0.0032174420932480005 equal +UD-Q5_K_XL FactChecking len02000 768 2406 2406 768 0.9998157041450705 0.002425756617665944 equal +UD-Q5_K_XL FactChecking len02000 512 2406 2406 768 0.9998260041647673 0.0027640632795226418 equal +UD-Q5_K_XL FactChecking len02000 256 2406 2406 768 0.9998440695296098 0.0034760011035124237 equal +UD-Q5_K_XL FactChecking len02000 128 2406 2406 768 0.9999119381535034 0.0031245425428631946 equal +UD-Q5_K_XL FactChecking len08000 768 11402 11402 768 0.9999294734092516 0.0015928794619447153 equal +UD-Q5_K_XL FactChecking len08000 512 11402 11402 768 0.9999319855463954 0.0013635772032167415 equal +UD-Q5_K_XL FactChecking len08000 256 11402 11402 768 0.9999440176583895 0.0017235962820425386 equal +UD-Q5_K_XL FactChecking len08000 128 11402 11402 768 0.9999664761649109 0.001900693017680794 equal +UD-Q5_K_XL CodeRetrieval len00008 768 15 15 768 0.9996809676724181 0.002966964447239276 equal +UD-Q5_K_XL CodeRetrieval len00008 512 15 15 768 0.9996738217970381 0.0034233485958491683 equal +UD-Q5_K_XL CodeRetrieval len00008 256 15 15 768 0.9997212382605235 0.004269061548825309 equal +UD-Q5_K_XL CodeRetrieval len00008 128 15 15 768 0.9998320633401426 0.004491302426813962 equal +UD-Q5_K_XL CodeRetrieval len00128 768 143 143 768 0.9997349146380865 0.002678643554352607 equal +UD-Q5_K_XL CodeRetrieval len00128 512 143 143 768 0.9997326641097836 0.0030725711807448753 equal +UD-Q5_K_XL CodeRetrieval len00128 256 143 143 768 0.9997490059307983 0.0038023946091755613 equal +UD-Q5_K_XL CodeRetrieval len00128 128 143 143 768 0.9998356422137431 0.004213664486903357 equal +UD-Q5_K_XL CodeRetrieval len01024 768 1148 1148 768 0.9998673885245632 0.0016663483348524112 equal +UD-Q5_K_XL CodeRetrieval len01024 512 1148 1148 768 0.9998742125829417 0.001907637767672058 equal +UD-Q5_K_XL CodeRetrieval len01024 256 1148 1148 768 0.9998966715500777 0.002331988082865527 equal +UD-Q5_K_XL CodeRetrieval len01024 128 1148 1148 768 0.9999300764020408 0.002496677502427283 equal +UD-Q5_K_XL CodeRetrieval len02000 768 2406 2406 768 0.9997728653398904 0.003214733995970276 equal +UD-Q5_K_XL CodeRetrieval len02000 512 2406 2406 768 0.9997877503168581 0.003674170832965274 equal +UD-Q5_K_XL CodeRetrieval len02000 256 2406 2406 768 0.9997944429787058 0.004595104334561052 equal +UD-Q5_K_XL CodeRetrieval len02000 128 2406 2406 768 0.999884077371837 0.0038000617406058756 equal +UD-Q5_K_XL CodeRetrieval len08000 768 11402 11402 768 0.9999248217593927 0.001388894532986882 equal +UD-Q5_K_XL CodeRetrieval len08000 512 11402 11402 768 0.9999280024858357 0.00158142425207633 equal +UD-Q5_K_XL CodeRetrieval len08000 256 11402 11402 768 0.9999400211453904 0.001988939358381153 equal +UD-Q5_K_XL CodeRetrieval len08000 128 11402 11402 768 0.9999611044341865 0.00210403316177425 equal +UD-Q5_K_XL Classification len00008 768 14 14 768 0.9992187628176539 0.004884101326646636 equal +UD-Q5_K_XL Classification len00008 512 14 14 768 0.9992038204539723 0.005628671319873066 equal +UD-Q5_K_XL Classification len00008 256 14 14 768 0.9992096114297733 0.007071713814021351 equal +UD-Q5_K_XL Classification len00008 128 14 14 768 0.9994840047558449 0.007320875241126137 equal +UD-Q5_K_XL Classification len00128 768 142 142 768 0.999778413462459 0.0023917970733093943 equal +UD-Q5_K_XL Classification len00128 512 142 142 768 0.9997808213782207 0.002455487637654921 equal +UD-Q5_K_XL Classification len00128 256 142 142 768 0.9998149883419843 0.0030198107659886696 equal +UD-Q5_K_XL Classification len00128 128 142 142 768 0.9998851095160548 0.0034643408614679627 equal +UD-Q5_K_XL Classification len01024 768 1147 1147 768 0.999831284958753 0.0020043839938805527 equal +UD-Q5_K_XL Classification len01024 512 1147 1147 768 0.9998396862941701 0.002293112879483833 equal +UD-Q5_K_XL Classification len01024 256 1147 1147 768 0.999861694470405 0.002872862222913327 equal +UD-Q5_K_XL Classification len01024 128 1147 1147 768 0.9999260732804599 0.0029322611050552616 equal +UD-Q5_K_XL Classification len02000 768 2405 2405 768 0.9998102777687615 0.002356156956461841 equal +UD-Q5_K_XL Classification len02000 512 2405 2405 768 0.9998231342718089 0.0026881138280446613 equal +UD-Q5_K_XL Classification len02000 256 2405 2405 768 0.9998326837139732 0.0033560128889180934 equal +UD-Q5_K_XL Classification len02000 128 2405 2405 768 0.9998856157629101 0.003793167427736526 equal +UD-Q5_K_XL Classification len08000 768 11401 11401 768 0.9999340971816398 0.001599317954006663 equal +UD-Q5_K_XL Classification len08000 512 11401 11401 768 0.9999365012168078 0.0018230346267741164 equal +UD-Q5_K_XL Classification len08000 256 11401 11401 768 0.99994463332268 0.002291423987194053 equal +UD-Q5_K_XL Classification len08000 128 11401 11401 768 0.9999664320123642 0.0024968400793019714 equal +UD-Q5_K_XL Clustering len00008 768 14 14 768 0.9997731219588297 0.0025268636401474115 equal +UD-Q5_K_XL Clustering len00008 512 14 14 768 0.9997783701490125 0.0028938222948572123 equal +UD-Q5_K_XL Clustering len00008 256 14 14 768 0.9997994207890704 0.003613845353148959 equal +UD-Q5_K_XL Clustering len00008 128 14 14 768 0.9998657106758869 0.00403119192497868 equal +UD-Q5_K_XL Clustering len00128 768 142 142 768 0.9998527551303155 0.0024111641847013275 equal +UD-Q5_K_XL Clustering len00128 512 142 142 768 0.9998519842018212 0.0027599098088733585 equal +UD-Q5_K_XL Clustering len00128 256 142 142 768 0.9998773758198402 0.0033760609466602035 equal +UD-Q5_K_XL Clustering len00128 128 142 142 768 0.999917164498848 0.0033590979008355504 equal +UD-Q5_K_XL Clustering len01024 768 1147 1147 768 0.999868360960938 0.0019307808259173033 equal +UD-Q5_K_XL Clustering len01024 512 1147 1147 768 0.999874524717298 0.0022094812136321113 equal +UD-Q5_K_XL Clustering len01024 256 1147 1147 768 0.9999035543261044 0.0025301693772026776 equal +UD-Q5_K_XL Clustering len01024 128 1147 1147 768 0.9999441714484965 0.0024603434827177324 equal +UD-Q5_K_XL Clustering len02000 768 2405 2405 768 0.9998338709926536 0.0022352998201754853 equal +UD-Q5_K_XL Clustering len02000 512 2405 2405 768 0.9998373550443871 0.002549694769094591 equal +UD-Q5_K_XL Clustering len02000 256 2405 2405 768 0.9998482237112918 0.0031387760562383727 equal +UD-Q5_K_XL Clustering len02000 128 2405 2405 768 0.9999063616519323 0.0027845123779042576 equal +UD-Q5_K_XL Clustering len08000 768 11401 11401 768 0.9999377377300701 0.0013709751812979107 equal +UD-Q5_K_XL Clustering len08000 512 11401 11401 768 0.9999424972665252 0.0015596496451150323 equal +UD-Q5_K_XL Clustering len08000 256 11401 11401 768 0.9999531544344736 0.001954543633657017 equal +UD-Q5_K_XL Clustering len08000 128 11401 11401 768 0.9999705645151808 0.0020726448410008647 equal +UD-Q5_K_XL SentenceSimilarity len00008 768 15 15 768 0.9998531880339685 0.002411062730551583 equal +UD-Q5_K_XL SentenceSimilarity len00008 512 15 15 768 0.9998561762428978 0.002769987236544976 equal +UD-Q5_K_XL SentenceSimilarity len00008 256 15 15 768 0.999868571745503 0.003467968805705046 equal +UD-Q5_K_XL SentenceSimilarity len00008 128 15 15 768 0.9999030566394664 0.00398005103920392 equal +UD-Q5_K_XL SentenceSimilarity len00128 768 143 143 768 0.9998915383702087 0.001785583356520376 equal +UD-Q5_K_XL SentenceSimilarity len00128 512 143 143 768 0.9998912396071481 0.002061370198479573 equal +UD-Q5_K_XL SentenceSimilarity len00128 256 143 143 768 0.9999076248262382 0.0026131435083608726 equal +UD-Q5_K_XL SentenceSimilarity len00128 128 143 143 768 0.9999432708664489 0.002599347565826459 equal +UD-Q5_K_XL SentenceSimilarity len01024 768 1148 1148 768 0.999875237663989 0.0018034353959013802 equal +UD-Q5_K_XL SentenceSimilarity len01024 512 1148 1148 768 0.9998777286282148 0.0020574826683733456 equal +UD-Q5_K_XL SentenceSimilarity len01024 256 1148 1148 768 0.9999020108913607 0.0025884792884960095 equal +UD-Q5_K_XL SentenceSimilarity len01024 128 1148 1148 768 0.9999456716216474 0.002938351017831166 equal +UD-Q5_K_XL SentenceSimilarity len02000 768 2406 2406 768 0.9998939071978303 0.0016528042172979478 equal +UD-Q5_K_XL SentenceSimilarity len02000 512 2406 2406 768 0.9998952627125505 0.0018859530687427226 equal +UD-Q5_K_XL SentenceSimilarity len02000 256 2406 2406 768 0.9999131402217457 0.0021403539346461706 equal +UD-Q5_K_XL SentenceSimilarity len02000 128 2406 2406 768 0.9999487105560908 0.002386282802119933 equal +UD-Q5_K_XL SentenceSimilarity len08000 768 11402 11402 768 0.9999462102832887 0.0012909978038577767 equal +UD-Q5_K_XL SentenceSimilarity len08000 512 11402 11402 768 0.9999494545655125 0.0014723932060089948 equal +UD-Q5_K_XL SentenceSimilarity len08000 256 11402 11402 768 0.9999559692507788 0.0018553246703137038 equal +UD-Q5_K_XL SentenceSimilarity len08000 128 11402 11402 768 0.9999730333640101 0.0017629766252187078 equal +UD-Q4_K_XL SearchQuery len00008 768 15 15 768 0.9995763591159268 0.003429484427103842 equal +UD-Q4_K_XL SearchQuery len00008 512 15 15 768 0.9995940857181507 0.003981098146850791 equal +UD-Q4_K_XL SearchQuery len00008 256 15 15 768 0.9996128950217985 0.004949961782088216 equal +UD-Q4_K_XL SearchQuery len00008 128 15 15 768 0.9997304909012146 0.0056745125388078105 equal +UD-Q4_K_XL SearchQuery len00128 768 143 143 768 0.9996845985909315 0.0032328720750139234 equal +UD-Q4_K_XL SearchQuery len00128 512 143 143 768 0.9996841023585735 0.0037238482063553235 equal +UD-Q4_K_XL SearchQuery len00128 256 143 143 768 0.9997100340292643 0.004554285113523376 equal +UD-Q4_K_XL SearchQuery len00128 128 143 143 768 0.9997596012666621 0.005606546250177619 equal +UD-Q4_K_XL SearchQuery len01024 768 1148 1148 768 0.9997737500567271 0.0027375034499751427 equal +UD-Q4_K_XL SearchQuery len01024 512 1148 1148 768 0.9997834102288568 0.0031222844573581647 equal +UD-Q4_K_XL SearchQuery len01024 256 1148 1148 768 0.9998167687376283 0.0038556739143335225 equal +UD-Q4_K_XL SearchQuery len01024 128 1148 1148 768 0.9998772343950688 0.004629398966414608 equal +UD-Q4_K_XL SearchQuery len02000 768 2406 2406 768 0.9997390266590322 0.0030284250841774024 equal +UD-Q4_K_XL SearchQuery len02000 512 2406 2406 768 0.9997344637387716 0.003465853404776731 equal +UD-Q4_K_XL SearchQuery len02000 256 2406 2406 768 0.9997493823085322 0.00430344785765506 equal +UD-Q4_K_XL SearchQuery len02000 128 2406 2406 768 0.9998175914370633 0.004957724635648572 equal +UD-Q4_K_XL SearchQuery len08000 768 11402 11402 768 0.9999286007906132 0.001686388166074644 equal +UD-Q4_K_XL SearchQuery len08000 512 11402 11402 768 0.9999292893809961 0.0019126044866137545 equal +UD-Q4_K_XL SearchQuery len08000 256 11402 11402 768 0.9999331945964637 0.002428871257531773 equal +UD-Q4_K_XL SearchQuery len08000 128 11402 11402 768 0.9999626423946897 0.001966747112397374 equal +UD-Q4_K_XL QuestionAnswering len00008 768 15 15 768 0.9992867849393449 0.005054687736705473 equal +UD-Q4_K_XL QuestionAnswering len00008 512 15 15 768 0.9992604502572765 0.005855689240444403 equal +UD-Q4_K_XL QuestionAnswering len00008 256 15 15 768 0.9993365581256991 0.00737282574052775 equal +UD-Q4_K_XL QuestionAnswering len00008 128 15 15 768 0.9995928547159669 0.008641196110932581 equal +UD-Q4_K_XL QuestionAnswering len00128 768 143 143 768 0.9993391838298232 0.004677412318518318 equal +UD-Q4_K_XL QuestionAnswering len00128 512 143 143 768 0.9993596817100058 0.005342165164450168 equal +UD-Q4_K_XL QuestionAnswering len00128 256 143 143 768 0.9994848421365198 0.006590486254062118 equal +UD-Q4_K_XL QuestionAnswering len00128 128 143 143 768 0.9996677404293747 0.00787169869466963 equal +UD-Q4_K_XL QuestionAnswering len01024 768 1148 1148 768 0.9997337399192237 0.00297425811907227 equal +UD-Q4_K_XL QuestionAnswering len01024 512 1148 1148 768 0.9997561494036056 0.0033917531174115787 equal +UD-Q4_K_XL QuestionAnswering len01024 256 1148 1148 768 0.9997903557258451 0.0034438773601584843 equal +UD-Q4_K_XL QuestionAnswering len01024 128 1148 1148 768 0.9998463569175343 0.004076722135077357 equal +UD-Q4_K_XL QuestionAnswering len02000 768 2406 2406 768 0.9996470284663834 0.003186449122664501 equal +UD-Q4_K_XL QuestionAnswering len02000 512 2406 2406 768 0.9996594824105091 0.0036382947363957493 equal +UD-Q4_K_XL QuestionAnswering len02000 256 2406 2406 768 0.999682912826178 0.004505988000457522 equal +UD-Q4_K_XL QuestionAnswering len02000 128 2406 2406 768 0.9998079247240917 0.00529186599349029 equal +UD-Q4_K_XL QuestionAnswering len08000 768 11402 11402 768 0.9999045889286332 0.0015155752412725106 equal +UD-Q4_K_XL QuestionAnswering len08000 512 11402 11402 768 0.999906086035773 0.0017255576137517907 equal +UD-Q4_K_XL QuestionAnswering len08000 256 11402 11402 768 0.9999146495495279 0.002167105328657914 equal +UD-Q4_K_XL QuestionAnswering len08000 128 11402 11402 768 0.9999495894632243 0.002477689448779747 equal +UD-Q4_K_XL FactChecking len00008 768 15 15 768 0.9992983018184873 0.004184258274556362 equal +UD-Q4_K_XL FactChecking len00008 512 15 15 768 0.9993059935131423 0.004830370433827854 equal +UD-Q4_K_XL FactChecking len00008 256 15 15 768 0.9993725855962237 0.0060642234355010385 equal +UD-Q4_K_XL FactChecking len00008 128 15 15 768 0.9995793192751746 0.006577559512253032 equal +UD-Q4_K_XL FactChecking len00128 768 143 143 768 0.999408387043497 0.004493390109977897 equal +UD-Q4_K_XL FactChecking len00128 512 143 143 768 0.9994317478968859 0.005158545481386242 equal +UD-Q4_K_XL FactChecking len00128 256 143 143 768 0.9995419917604202 0.006410401233294866 equal +UD-Q4_K_XL FactChecking len00128 128 143 143 768 0.9996605009934605 0.007705193623367945 equal +UD-Q4_K_XL FactChecking len01024 768 1148 1148 768 0.9997168889238949 0.0032422042219211883 equal +UD-Q4_K_XL FactChecking len01024 512 1148 1148 768 0.999723928668614 0.0036934605568570145 equal +UD-Q4_K_XL FactChecking len01024 256 1148 1148 768 0.9997627432537927 0.004544616710291105 equal +UD-Q4_K_XL FactChecking len01024 128 1148 1148 768 0.9998323175659805 0.004694900135804786 equal +UD-Q4_K_XL FactChecking len02000 768 2406 2406 768 0.9996959033911347 0.0033731604010851235 equal +UD-Q4_K_XL FactChecking len02000 512 2406 2406 768 0.9997059288408376 0.003842747034855487 equal +UD-Q4_K_XL FactChecking len02000 256 2406 2406 768 0.9997382426529583 0.004722884384726345 equal +UD-Q4_K_XL FactChecking len02000 128 2406 2406 768 0.9998133187733578 0.005010672965994445 equal +UD-Q4_K_XL FactChecking len08000 768 11402 11402 768 0.9999238109453599 0.0018671477812337023 equal +UD-Q4_K_XL FactChecking len08000 512 11402 11402 768 0.9999285335331516 0.002133909977843358 equal +UD-Q4_K_XL FactChecking len08000 256 11402 11402 768 0.9999378091657578 0.002018889645598529 equal +UD-Q4_K_XL FactChecking len08000 128 11402 11402 768 0.9999594924472787 0.0020004535369734077 equal +UD-Q4_K_XL CodeRetrieval len00008 768 15 15 768 0.9996012184020396 0.003536711896923001 equal +UD-Q4_K_XL CodeRetrieval len00008 512 15 15 768 0.9996049406804588 0.004064391961826397 equal +UD-Q4_K_XL CodeRetrieval len00008 256 15 15 768 0.9996511268183332 0.0048032594847759544 equal +UD-Q4_K_XL CodeRetrieval len00008 128 15 15 768 0.999813350845179 0.004511026059051501 equal +UD-Q4_K_XL CodeRetrieval len00128 768 143 143 768 0.9996008667423399 0.0032888902185629846 equal +UD-Q4_K_XL CodeRetrieval len00128 512 143 143 768 0.9996083226118063 0.0037700562641223116 equal +UD-Q4_K_XL CodeRetrieval len00128 256 143 143 768 0.9996338830648187 0.0046316588658828 equal +UD-Q4_K_XL CodeRetrieval len00128 128 143 143 768 0.9997934189461594 0.005095478086656613 equal +UD-Q4_K_XL CodeRetrieval len01024 768 1148 1148 768 0.9997140259924763 0.0028320238753751573 equal +UD-Q4_K_XL CodeRetrieval len01024 512 1148 1148 768 0.9997326991881483 0.0029964401940292193 equal +UD-Q4_K_XL CodeRetrieval len01024 256 1148 1148 768 0.9997573485857124 0.003694183199474989 equal +UD-Q4_K_XL CodeRetrieval len01024 128 1148 1148 768 0.999825508345072 0.003926414540521346 equal +UD-Q4_K_XL CodeRetrieval len02000 768 2406 2406 768 0.999691058006733 0.003351090416013936 equal +UD-Q4_K_XL CodeRetrieval len02000 512 2406 2406 768 0.9996933091987078 0.003825267708639009 equal +UD-Q4_K_XL CodeRetrieval len02000 256 2406 2406 768 0.9997158241374434 0.004772936711255438 equal +UD-Q4_K_XL CodeRetrieval len02000 128 2406 2406 768 0.9997904992574457 0.005482519791714571 equal +UD-Q4_K_XL CodeRetrieval len08000 768 11402 11402 768 0.9999342653596461 0.0013088136113053352 equal +UD-Q4_K_XL CodeRetrieval len08000 512 11402 11402 768 0.9999366116778206 0.0014893860225716274 equal +UD-Q4_K_XL CodeRetrieval len08000 256 11402 11402 768 0.9999425543829066 0.001860517089296524 equal +UD-Q4_K_XL CodeRetrieval len08000 128 11402 11402 768 0.999963739973386 0.002127792068312019 equal +UD-Q4_K_XL Classification len00008 768 14 14 768 0.9993248616599824 0.003981372990757851 equal +UD-Q4_K_XL Classification len00008 512 14 14 768 0.9993270270216671 0.004577690650287302 equal +UD-Q4_K_XL Classification len00008 256 14 14 768 0.9993749862719086 0.005737825917778841 equal +UD-Q4_K_XL Classification len00008 128 14 14 768 0.9995502599562308 0.006537949509018974 equal +UD-Q4_K_XL Classification len00128 768 142 142 768 0.9996387382113255 0.002960871728081127 equal +UD-Q4_K_XL Classification len00128 512 142 142 768 0.9996395397784978 0.003381217102754717 equal +UD-Q4_K_XL Classification len00128 256 142 142 768 0.9997017603403433 0.004180715506321125 equal +UD-Q4_K_XL Classification len00128 128 142 142 768 0.9998095663187423 0.004860121281257509 equal +UD-Q4_K_XL Classification len01024 768 1147 1147 768 0.9997088122838405 0.0027696143188415782 equal +UD-Q4_K_XL Classification len01024 512 1147 1147 768 0.9997138327991499 0.0031521696811598297 equal +UD-Q4_K_XL Classification len01024 256 1147 1147 768 0.9997468677204928 0.003926716494109209 equal +UD-Q4_K_XL Classification len01024 128 1147 1147 768 0.999835145662103 0.004475436487964217 equal +UD-Q4_K_XL Classification len02000 768 2405 2405 768 0.9996883606318906 0.004185393314813046 equal +UD-Q4_K_XL Classification len02000 512 2405 2405 768 0.9996981363689287 0.004806427976766836 equal +UD-Q4_K_XL Classification len02000 256 2405 2405 768 0.9997420859295186 0.006002826032854625 equal +UD-Q4_K_XL Classification len02000 128 2405 2405 768 0.9998080473081364 0.0070531555071511876 equal +UD-Q4_K_XL Classification len08000 768 11401 11401 768 0.9999280122045187 0.001564148070211921 equal +UD-Q4_K_XL Classification len08000 512 11401 11401 768 0.9999282692590975 0.0017842468405116427 equal +UD-Q4_K_XL Classification len08000 256 11401 11401 768 0.9999334895344208 0.0022377861031488942 equal +UD-Q4_K_XL Classification len08000 128 11401 11401 768 0.9999516094319749 0.0025507340933678888 equal +UD-Q4_K_XL Clustering len00008 768 14 14 768 0.9997953503662166 0.0026427798400445422 equal +UD-Q4_K_XL Clustering len00008 512 14 14 768 0.9998019246856914 0.0030374635960769766 equal +UD-Q4_K_XL Clustering len00008 256 14 14 768 0.9998181370197311 0.0037809454138256447 equal +UD-Q4_K_XL Clustering len00008 128 14 14 768 0.9998772100782966 0.004257378381553489 equal +UD-Q4_K_XL Clustering len00128 768 142 142 768 0.9997207646392381 0.002765276022906542 equal +UD-Q4_K_XL Clustering len00128 512 142 142 768 0.9997431027847492 0.0031567965332998747 equal +UD-Q4_K_XL Clustering len00128 256 142 142 768 0.9998001547632884 0.003813815540325366 equal +UD-Q4_K_XL Clustering len00128 128 142 142 768 0.9998580318873568 0.00358795185886919 equal +UD-Q4_K_XL Clustering len01024 768 1147 1147 768 0.9997738789875282 0.002884969717208259 equal +UD-Q4_K_XL Clustering len01024 512 1147 1147 768 0.9997847936308157 0.0032982510828841044 equal +UD-Q4_K_XL Clustering len01024 256 1147 1147 768 0.9998217256198634 0.004002970150544502 equal +UD-Q4_K_XL Clustering len01024 128 1147 1147 768 0.9998913090466748 0.004502719413127368 equal +UD-Q4_K_XL Clustering len02000 768 2405 2405 768 0.9997156499894857 0.003069858237734306 equal +UD-Q4_K_XL Clustering len02000 512 2405 2405 768 0.999736178637785 0.0034967767153636156 equal +UD-Q4_K_XL Clustering len02000 256 2405 2405 768 0.9997771382279452 0.0043080044639981566 equal +UD-Q4_K_XL Clustering len02000 128 2405 2405 768 0.9998520733880606 0.004860758851977266 equal +UD-Q4_K_XL Clustering len08000 768 11401 11401 768 0.999920967400047 0.0015201338014746782 equal +UD-Q4_K_XL Clustering len08000 512 11401 11401 768 0.9999208467970104 0.0017285462083773268 equal +UD-Q4_K_XL Clustering len08000 256 11401 11401 768 0.9999282598117831 0.002158159142397517 equal +UD-Q4_K_XL Clustering len08000 128 11401 11401 768 0.9999503397539912 0.0021631826846558316 equal +UD-Q4_K_XL SentenceSimilarity len00008 768 15 15 768 0.9997499518166676 0.0026052846402235263 equal +UD-Q4_K_XL SentenceSimilarity len00008 512 15 15 768 0.9997473126854225 0.0030094812223849868 equal +UD-Q4_K_XL SentenceSimilarity len00008 256 15 15 768 0.9998039890346121 0.0034919169720669455 equal +UD-Q4_K_XL SentenceSimilarity len00008 128 15 15 768 0.9998563711327008 0.004030298483651426 equal +UD-Q4_K_XL SentenceSimilarity len00128 768 143 143 768 0.9998428961260497 0.0021055450351830617 equal +UD-Q4_K_XL SentenceSimilarity len00128 512 143 143 768 0.9998408399225884 0.0024232168550407385 equal +UD-Q4_K_XL SentenceSimilarity len00128 256 143 143 768 0.9998617069679742 0.0030384865278817418 equal +UD-Q4_K_XL SentenceSimilarity len00128 128 143 143 768 0.9999165685919065 0.003090533009814187 equal +UD-Q4_K_XL SentenceSimilarity len01024 768 1148 1148 768 0.9998461284336383 0.0020524617446291277 equal +UD-Q4_K_XL SentenceSimilarity len01024 512 1148 1148 768 0.9998513809114707 0.0023389151706867534 equal +UD-Q4_K_XL SentenceSimilarity len01024 256 1148 1148 768 0.9998649586970803 0.002900885605554071 equal +UD-Q4_K_XL SentenceSimilarity len01024 128 1148 1148 768 0.9998878298854289 0.003398999319933191 equal +UD-Q4_K_XL SentenceSimilarity len02000 768 2406 2406 768 0.9998265496266621 0.0020209516621603196 equal +UD-Q4_K_XL SentenceSimilarity len02000 512 2406 2406 768 0.9998279071490881 0.0023148417338412584 equal +UD-Q4_K_XL SentenceSimilarity len02000 256 2406 2406 768 0.9998669158940291 0.002872150423139083 equal +UD-Q4_K_XL SentenceSimilarity len02000 128 2406 2406 768 0.9999136240082911 0.0032987891814067125 equal +UD-Q4_K_XL SentenceSimilarity len08000 768 11402 11402 768 0.9999260168316085 0.0019471826253367498 equal +UD-Q4_K_XL SentenceSimilarity len08000 512 11402 11402 768 0.9999287296972457 0.0022244182938744896 equal +UD-Q4_K_XL SentenceSimilarity len08000 256 11402 11402 768 0.9999353768395625 0.0027836685434584167 equal +UD-Q4_K_XL SentenceSimilarity len08000 128 11402 11402 768 0.9999552980415303 0.0031846464639658442 equal diff --git a/docs/audits/APR-EMBED-001/EG-2/parity.md b/docs/audits/APR-EMBED-001/EG-2/parity.md new file mode 100644 index 0000000000..c6f7328492 --- /dev/null +++ b/docs/audits/APR-EMBED-001/EG-2/parity.md @@ -0,0 +1,247 @@ +# EG-2 forward parity: provisional receipt (2026-10-10) + +**Result: PASS (provisional): 840 of 840 I-8 cells at or above 0.98, 0 token mismatches; I-4 and I-6 PASS; every plant RED** + +| Item | Value | +|------|-------| +| apr | release build of `feat/apr-embed-001-eg1` @ `f112cf6bc7` plus the EG-2 changes of this commit, sha256 `3979204bbe364bfa9c16252bffeaabd286cf7c1968294291247b20fd35e1d1c1` | +| Oracle | llama.cpp b11490 (`9c2e0e491`) `llama-embedding --embd-normalize 2`, CPU build, sha256 `cc4d0223cedccf489e245f4505b9bdeaae4ed807105964ca16881e46bf177709`. **PROVISIONAL** (§7) | +| Pinned oracle | `scripts/llama_pin.toml` build_commit `d1d3c3396` refuses the file: `unknown model architecture: 'gemma-embedding2'`. Moving the pin is D-3, a §7 pin-change STOP, so it was not moved | +| Files | unsloth BF16, F16, Q8_0, UD-Q4_K_XL, UD-Q5_K_XL, UD-Q6_K_XL (6 of 6) | +| Prompts | the 7 query prompts of `corpus/prompts.tsv`, applied by `apr embed --task ` and by template for the oracle | +| Lengths | the 5 texts of `corpus/lengths.tsv`: 8, 128, 1024, 2000 and 8000 tokens. The local window is 1024 (512 each side), so the last three are the ones where the local and global masks differ | +| Dimensions | 768, 512, 256, 128: the cosine of the first d components of each vector. A cosine does not see scale, so this is the cosine of the re-normalised prefixes | +| Cells | 6 files × 7 prompts × 5 lengths × 4 dimensions = 840 | +| Threads | 4 for both, on one CPU host | + +**Which tree the binaries came from.** Every binary, test and lint below came from one copy of the tree. Before +the first build, all 28,052 files of the branch were compared with that copy by checksum: the committed tree, plus +the EG-2 files this commit adds. They matched, except for the two EG-1 receipt files, which nothing builds. The +comparison was repeated before the last steps, after the plant patch had been reverted. The parity cells ran on +one release build, `apr 0.70.2 (f112cf6bc7-eg2fix)`, sha256 `3979204bbe364bfa9c16252bffeaabd286cf7c1968294291247b20fd35e1d1c1`. + +**One move after the runs.** The commit hook refused `attention.rs` for the complexity of a function this +branch does not touch (`tiled_single_head_attention`). The windowed attention this branch added there moved, +unchanged, into its own file, `attention_windowed.rs`, and `attention.rs` is now as it was before the branch. +The moved tree was compared with the commit by checksum (28,055 files, all equal but the four receipt files, +which nothing builds). On it, the release build, the serve tests (1275 passed, 0 failed, 8 ignored) and both +serve clippy steps were run again and passed. Its `apr` (sha256 +`18c6c38023b768e8610343135984bf70d8ed29989db8a88680901e210ae84606`) gave vectors bit-equal to the parity +build's on all 5 lengths, for BF16 with the classification prompt and UD-Q4_K_XL with the search-query prompt. +The other cells were not run again. + +A first run of the cells, on a build of the forward before the shape fix (finding 8), is void. That build refused +every real file at load. Its records are kept, and none of its numbers are used here. The first oracle side of +the BF16 cells is void too: it ran before the oracle was given `--no-escape` (finding 9). The BF16 oracle cells +were run again with the flag against the same `apr` vectors, and only that second run is used. + +## I-8: `apr` against the oracle + +Each row is one file at one dimension, over 7 prompts × 5 lengths = 35 cells. A token mismatch is a cell where +`apr`'s token count differs from the oracle's `number of tokens in prompt`. The last column counts cells where +the prompt the oracle logged is not, byte for byte, the prompt it was given (finding 9). Every cell is in `i8-cells.tsv` beside this file. + +| File | Dim | Cells | Min cosine | Max \|Δ\| | Token mismatches | Oracle width ≠ 768 | Oracle prompt ≠ given | +|------|-----|-------|------------|-----------|------------------|--------------------|-----------------------| +| BF16 | 768 | 35 | 0.9999902 | 0.0005569 | 0 | 0 | 0 | +| BF16 | 512 | 35 | 0.9999901 | 0.0006422 | 0 | 0 | 0 | +| BF16 | 256 | 35 | 0.9999909 | 0.0007242 | 0 | 0 | 0 | +| BF16 | 128 | 35 | 0.9999942 | 0.0008267 | 0 | 0 | 0 | +| F16 | 768 | 35 | 0.9999994 | 0.0001378 | 0 | 0 | 0 | +| F16 | 512 | 35 | 0.9999994 | 0.0001607 | 0 | 0 | 0 | +| F16 | 256 | 35 | 0.9999995 | 0.0002031 | 0 | 0 | 0 | +| F16 | 128 | 35 | 0.9999996 | 0.0002241 | 0 | 0 | 0 | +| Q8_0 | 768 | 35 | 0.9997716 | 0.0027673 | 0 | 0 | 0 | +| Q8_0 | 512 | 35 | 0.9997802 | 0.0031852 | 0 | 0 | 0 | +| Q8_0 | 256 | 35 | 0.9997934 | 0.0039852 | 0 | 0 | 0 | +| Q8_0 | 128 | 35 | 0.9998300 | 0.0045931 | 0 | 0 | 0 | +| UD-Q6_K_XL | 768 | 35 | 0.9992352 | 0.0045409 | 0 | 0 | 0 | +| UD-Q6_K_XL | 512 | 35 | 0.9992453 | 0.0052333 | 0 | 0 | 0 | +| UD-Q6_K_XL | 256 | 35 | 0.9993490 | 0.0063002 | 0 | 0 | 0 | +| UD-Q6_K_XL | 128 | 35 | 0.9995339 | 0.0073799 | 0 | 0 | 0 | +| UD-Q5_K_XL | 768 | 35 | 0.9988872 | 0.0050412 | 0 | 0 | 0 | +| UD-Q5_K_XL | 512 | 35 | 0.9988615 | 0.0058222 | 0 | 0 | 0 | +| UD-Q5_K_XL | 256 | 35 | 0.9989918 | 0.0072789 | 0 | 0 | 0 | +| UD-Q5_K_XL | 128 | 35 | 0.9993532 | 0.0081475 | 0 | 0 | 0 | +| UD-Q4_K_XL | 768 | 35 | 0.9992868 | 0.0050547 | 0 | 0 | 0 | +| UD-Q4_K_XL | 512 | 35 | 0.9992605 | 0.0058557 | 0 | 0 | 0 | +| UD-Q4_K_XL | 256 | 35 | 0.9993366 | 0.0073728 | 0 | 0 | 0 | +| UD-Q4_K_XL | 128 | 35 | 0.9995503 | 0.0086412 | 0 | 0 | 0 | + +## The threshold, from known-good and known-bad pairs + +| Pair | n | Min cosine | Max cosine | +|------|---|------------|------------| +| oracle vs itself (noise floor), BF16, SearchQuery | 5 lengths | 1.0000000 | 1.0000000 | +| `apr` vs oracle (known-good), all files | 840 cells | 0.9988615 | — | +| `apr` vs oracle (known-good), planted binary with no plant (`none`), BF16 | 5 lengths | 0.9999971 | 0.9999993 | +| known-bad: the document form in place of the query prompt | 5 lengths | 0.9630706 | 0.9946302 | +| known-bad: a causal mask (`d-causal`) | 5 lengths | 0.8028001 | 0.9677693 | +| known-bad: the SiLU gate in place of GELU (`e-silu`) | 5 lengths | 0.9391608 | 0.9755112 | +| known-bad: the local-to-global pattern shifted by one layer (`f-shift`) | 5 lengths | 0.9553109 | 0.9993064 | + +The known-bad rows by length, at 768: + +| Pair | 8 | 128 | 1024 | 2000 | 8000 | +|------|---|-----|------|------|------| +| document form | 0.9699312 | 0.9630706 | 0.9844369 | 0.9837832 | 0.9946302 | +| `d-causal` | 0.8028001 | 0.8525408 | 0.9119064 | 0.9308720 | 0.9677693 | +| `e-silu` | 0.9755112 | 0.9588616 | 0.9391608 | 0.9408457 | 0.9481853 | +| `f-shift` | 0.9993064 | 0.9953435 | 0.9722698 | 0.9553109 | 0.9750457 | + +All 840 known-good cells are at or above 0.98, so I-8 passes at G11's floor. The two ranges overlap, +though (finding 10). The lowest known-good cell is 0.9988615 (UD-Q5_K_XL, FactChecking, 8 tokens, 512), and +the highest known-bad cell is 0.9993064 (`f-shift`, 8 tokens). 12 of the 840 known-good cells sit below that +known-bad maximum: 2 on UD-Q6_K_XL, 6 on UD-Q5_K_XL and 4 on UD-Q4_K_XL. On BF16, F16 and Q8_0 none do: their +lowest cell is 0.9997716. So no one floor separates known-good from known-bad across all six files. The +known-bad pairs were run on BF16 only, so a floor for a quantized file needs known-bad pairs on that file. + +The receipt line in the spec's form (§8): + +```text +EG-2 | PASS (provisional: oracle b11490 9c2e0e491, not the pin) | files 6/6 (BF16 F16 Q8_0 UD-Q6_K_XL UD-Q5_K_XL UD-Q4_K_XL) +parity: I-8 840/840 cells >= 0.98 (G11), min 0.9988615, token mismatches 0, oracle width != 768 0, oracle prompt != given 0; GPU vs CPU not_measured +invar: I-4 7/7 PASS, I-6 2/2 PASS (BF16, CLI); plants 7/7 as expected (none green, 6 RED); host thresholds.yaml entry not_measured (D6) +``` + +## I-4 and I-6 at the CLI (BF16) + +| Cell | Case | Value | Verdict | +|------|------|-------|---------| +| I-4 | default: header dim, then each vector length | `768 768,768` | PASS | +| I-4 | `--dim 512` text 0: len 512, norm 1.0000000, max\|d\| vs re-normalised prefix | 1.02e-08 | PASS | +| I-4 | `--dim 512` text 1: len 512, norm 1.0000000, max\|d\| vs re-normalised prefix | 7.06e-09 | PASS | +| I-4 | `--dim 256` text 0: len 256, norm 1.0000000, max\|d\| vs re-normalised prefix | 8.59e-09 | PASS | +| I-4 | `--dim 256` text 1: len 256, norm 1.0000000, max\|d\| vs re-normalised prefix | 1.08e-08 | PASS | +| I-4 | `--dim 128` text 0: len 128, norm 1.0000000, max\|d\| vs re-normalised prefix | 8.77e-09 | PASS | +| I-4 | `--dim 128` text 1: len 128, norm 1.0000000, max\|d\| vs re-normalised prefix | 9.63e-09 | PASS | +| I-6 | forward vs reversed batch of 8, bit-equal | 8/8 | PASS | +| I-6 | text 3 alone vs at position 3, bit-equal (max\|d\|) | 0 | PASS | + +The oracle calls of this step, for the noise floor and the document form, all logged the prompt they were given +(10 of 10). + +## The plants, observed RED (§9) + +One planted `apr` (sha256 `2eee6a0865bda30f8cd254b83bbcece78b3f33da9b391aec74d360208b83b8ab`) holds every plant, +chosen at run time by `APR_EG2_PLANT`. It was built from the same tree +with a patch that was reverted after the build. The three patched files were then compared with their pre-patch +checksums, byte for byte. The plants were never committed. The `none` row is the same planted binary with no +plant selected, and it must be green. Its I-8 cells first ran before the oracle had `--no-escape` (finding 9). +They were run again with the flag, and only that second run is used. + +| Plant | Invariant | What it does | Cell that went RED | Value | +|-------|-----------|--------------|--------------------|-------| +| `none` | all | no plant selected | none: the 9 CLI checks PASS, and I-8 is 0.9999971 to 0.9999993 over the 5 lengths | green | +| `a-trunk` | I-4 | `--dim 512` answers the pooled trunk state | `--dim 512`, texts 0 and 1: max\|d\| vs the re-normalised prefix | 0.3755542, 0.3328974 | +| `b-norm` | I-4 | the truncated prefix is not re-normalised | all 6 `--dim` rows: the norm | 0.578 to 0.867 | +| `c-pad` | I-6 | every text is padded with token 0 to the batch's longest, and pooled over it | text 3 alone vs at position 3 | max\|d\| 0.0182079 | +| `d-causal` | I-8 | a causal mask on top of the window | all 5 lengths | 0.8028001 to 0.9677693 | +| `e-silu` | I-8 | the SiLU gate in place of GELU | all 5 lengths | 0.9391608 to 0.9755112 | +| `f-shift` | I-8 | each layer takes its window and RoPE base from the layer before | 1024, 2000 and 8000 | 0.9722698, 0.9553109, 0.9750457 | + +Every plant went RED in at least one cell, and the oracle logged the prompt it was given in all 20 of the plants' +oracle calls. Two plants are not RED everywhere, which is why each check exists: + +- `c-pad` passes the reversed-batch check (8/8 bit-equal): padding every text to the same longest text is + symmetric under reversal. Only the check of one text alone against the same text inside the batch sees it. +- `f-shift` stays above the floor at 8 and 128 tokens (0.9993064, 0.9953435). On a text shorter than the half + window of 512, the local and global masks are the same, so only the RoPE base moves, and at short range that + moves the vector little. The lengths of 1024 tokens and over are the ones that catch it. + +## Tests and lints + +Each step ran with `-j4` on one CPU host, on the copy of the tree described above. + +| Step | Command | Result | +|------|---------|--------| +| release build | `cargo build --release --bin apr` | exit 0 | +| core tests | `cargo test -p aprender-core --lib -- test_4900 test_3601 test_3656 test_3947` | 16 passed, 0 failed | +| core clippy | `cargo clippy -p aprender-core --lib -- -D warnings` | exit 0 | +| serve tests | `cargo test -p aprender-serve --lib -- eg2 matmul get_tensor_f32 dequant` | 1275 passed, 0 failed, 8 ignored | +| serve clippy | `cargo clippy -p aprender-serve --lib -- -D warnings` | exit 0 | +| serve clippy, test profile | `cargo clippy -p aprender-serve --lib --profile test -- -D warnings` | exit 0 | +| cli tests | `cargo test -p apr-cli --lib -- embed model_kind` (includes the BERT `apr embed` tests) | 178 passed, 0 failed | +| cli clippy | `cargo clippy -p apr-cli --lib -- -D warnings` | exit 0 | + +The 8 ignored serve tests carry `#[ignore]` in files this branch does not touch (`apr/apr_tests*.rs`, +`gguf/loader`, `quantize/tests`): three dtype reads, one real-model load, and four timing tests. They did not +run, so they are not measured here. 54 of the passed serve tests are this branch's `eg2` tests. + +Earlier runs on older copies of the tree failed on this branch's own code, and each failure was fixed before the +run above. The first serve run had 1271 passed and 2 failed, both in this branch's own tests: + +- `every_tensor_reaches_the_embedding_eg2` stopped at the first tensor that did not move the output, and reported + no number. It now collects every such tensor with its cosine, so one run shows all of them. It passes in the run above, after the fixture fix of finding 7. +- `truncation_keeps_the_prefix_direction_at_unit_length_eg2` compared the full-width truncation with the input + exactly. At full width, the vector is scaled by 1 / |v|, which is 1 only to within rounding. The test now allows + a change of at most 1e-6 per component. + +The same run's clippy reported three `needless_for_each` findings in `forward_embedding_gemma2.rs`. Each closure +became a `for` loop (`*v *= inv` in both forms). Findings 7 to 10 below came from later runs. + +## Findings + +1. **The census FFN pattern cannot match a real call.** The first census command in + `docs/specifications/EPIC-0.74-any-model-plan.md` §6 (one of the two D8 uses) counts FFN calls with + `ffn_block::|adaptive_ffn::`. Those two files are pulled into `forward/results.rs` with `include!`, so no module of + either name exists and no call can carry the prefix; the only match in the crate is a doc comment. Its FFN count is + 0 for every forward file, whatever the file calls. +2. **The contract's formulas were wrong in two places, fixed in `embeddinggemma2-forward-v1.yaml` v1.1.0.** The + score scale is 1.0, not 1/√d_head, as in the reference forward. The local window is centred: |i − j| ≤ ⌊W/2⌋, + not a one-sided window of W. The unit test checks the boundary pair ⌊W/2⌋ (inside) and ⌊W/2⌋ + 1 (outside). +3. **The existing Gemma FFN and dispatch cover Gemma 1 and 2 only** (`geglu_ffn`, `gemma_dispatch`), so the + EmbeddingGemma 2 forward has its own GELU-gated FFN. +4. **A BERT APR file without `--vocab` now exits 5, not 2.** `--vocab` was a required flag, so clap refused its + absence with a usage error (exit 2). A GGUF file carries its own tokenizer, so the flag is optional now, and the + APR path refuses its absence itself as a validation failure (exit 5) with the reason in the message. +5. **The first I-4 check of the default length could not fail on a short vector.** It read the header's `dim` + field only. It now also reads each vector's length, and passes only on `768 768,768`. +6. **Clippy under `--profile test` is not clean on two crates, for code this branch does not touch.** On + `aprender-core` it reported 20 findings: 19 are in files this branch does not touch, so `main` has them too and + the step cannot be a merge gate. The one in a file this branch touches (`verbose_bit_mask` in the BF16 test) is + fixed in de03bbb402. On `apr-cli` it reported two `unused_mut` in `nf4_classifier.rs`, which this branch does not + touch. For clippy on the lib targets, see the tests section. +7. **The fixture's weights were rank 2, so the reach test could not see a weight.** The fixture filled each weight + with `sin(a·i + b)` along its flattened index. For a matrix that is `sin(A_row + B_col)`, which is rank 2. With + every weight rank 2, the fixture's embeddings all lay on one circle, and a perturbed weight could move the vector + by less than the cosine can resolve. The fixture now fills weights from an integer hash of the index, and + `the_fixture_embeds_beyond_a_plane_eg2` requires three embeddings to have a Gram determinant above 1e-4. +8. **The forward read every weight's shape the wrong way round, and its fixture agreed with it.** The GGUF reader + in `aprender-serve` reverses GGUF's `ne0, ne1`, so it holds a weight as `[out, in]`. The generative config reads + shapes this way (C-13). The EG-2 forward read `[in, out]`, and the test fixture wrote its shapes the same wrong + way, so every unit test passed. Only the real file showed it. Each `attn_q` read back with an output width of + 512, the hidden width, so the forward found no local/global split and refused the file at load. That voided the + first run of the cells. Now the forward reads `[out, in]` and the fixture writes shapes as a real file holds them. + `the_fixture_holds_shapes_as_a_real_file_does_eg2` checks the fixture against the reader. On the real file, + `attn_q` is 1024 wide on the local layers and 2048 wide on the global layers (5, 11, 17, 23), so the split by + width holds. +9. **The oracle turned escape sequences in the text into bytes, so it embedded a different text from `apr`.** + `llama-embedding` reads `\x2c`, `\\`, `\r` and the like in `-p` as the bytes they name unless it is given + `--no-escape`. The 8000-token text quotes a tokenizer's escape table and holds seven backslashes. The other four + texts and the seven prompt templates hold none. On BF16 with the query prompt, `apr` counted 11402 tokens for that + text and the oracle 11393. The cosine was still 0.99990 at 768, well above the floor, so the cosine alone would + have passed a comparison of two different texts. The token count is what showed it. The prompt the oracle logged + first differed from the one it was given at the first `\x2c`. The oracle now runs with `--no-escape`. Each cell + also checks the prompt the oracle logs against the prompt it was given (the last column of the I-8 table). One + short text run both ways shows the flag's effect. With escapes on, `a\x2cb \\ c` became 7 tokens and the oracle + logged `a,b \ c`. With `--no-escape` it became 9 tokens and the oracle logged the text as written. +10. **At the 0.98 floor, 5 of the 20 known-bad I-8 cells pass.** The document form passes at 1024, 2000 and 8000 + tokens (0.9844369, 0.9837832, 0.9946302): the prompt is a few tokens at the head of a long text, and pooling + over the whole text dilutes it. `f-shift` passes at 8 and 128 tokens (see the plants). Each known-bad pair + still fails at some length, so the five lengths together catch all four, but no one length does. The + known-good and known-bad ranges overlap. The lowest known-good cell is 0.9988615, and the highest known-bad + cell is 0.9993064. 12 of the 840 known-good cells sit below it, all on the K-quant files (UD-Q6_K_XL 2, + UD-Q5_K_XL 6, UD-Q4_K_XL 4), and all on the 8-token text, at 256 to 768 dimensions. On BF16, F16 and Q8_0 the lowest + known-good cell is 0.9997716, above every known-bad cell. So no one floor separates the two across all six + files: a floor would need known-bad pairs measured on each quantized file (D6). This receipt does not move + G11: that is a gate change. + +## Why this is not the gate receipt + +- The oracle is b11490, not the pinned commit, which refuses the architecture. The same files, prompts, lengths and + dimensions become the gate receipt once a pin that loads `gemma-embedding2` is approved (D-3). +- The spec sets the threshold by the thresholds file's own method, on each required host. This is one CPU host, so + `evidence/parity/thresholds.yaml` gets no `models:` entry from it (D6). Until it has one, G11 holds the cosine + floor at 0.98, fail-closed. +- GPU against CPU is not measured: both sides ran on the CPU. From acea84528b104e2adb4799db90bb53db880bebed Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 18:16:32 +0200 Subject: [PATCH 16/21] =?UTF-8?q?fix(contracts):=20EG=20contracts=20pass?= =?UTF-8?q?=20the=20=CE=A3=20and=20strict=20test-binding=20gates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EG2-INV-002 named the local half window with a floor glyph that Σ does not declare; it is now h, defined in the property. FALSIFY-EG2L-007 cited `falsify_eg2l_007_*`; the binding gate matches a bare name as a substring without trimming the star, so it resolved to nothing. It now cites the prefix, which is also the filter `cargo test` takes. FALSIFY-EG2L-008 cites the spm_bpe tests module instead of a trailing `::*`. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- contracts/embeddinggemma2-forward-v1.yaml | 4 ++-- contracts/embeddinggemma2-load-v1.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contracts/embeddinggemma2-forward-v1.yaml b/contracts/embeddinggemma2-forward-v1.yaml index 0fc6a2bf01..c41ed6d960 100644 --- a/contracts/embeddinggemma2-forward-v1.yaml +++ b/contracts/embeddinggemma2-forward-v1.yaml @@ -112,8 +112,8 @@ proof_obligations: applies_to: all - id: EG2-INV-002 type: invariant - property: Local attention window - formal: "∀ l ∈ local, ∀ i, j: abs(i - j) > ⌊W/2⌋ ⇒ weight(i, j) = 0 ∧ abs(i - j) ≤ ⌊W/2⌋ ⇒ weight(i, j) > 0" + property: Local attention window, half width h = W / 2 rounded down + formal: "∀ l ∈ local, ∀ i, j: abs(i - j) > h ⇒ weight(i, j) = 0 ∧ abs(i - j) ≤ h ⇒ weight(i, j) > 0" applies_to: all - id: EG2-INV-003 type: invariant diff --git a/contracts/embeddinggemma2-load-v1.yaml b/contracts/embeddinggemma2-load-v1.yaml index bcd2ba7043..6a69c57c8b 100644 --- a/contracts/embeddinggemma2-load-v1.yaml +++ b/contracts/embeddinggemma2-load-v1.yaml @@ -141,12 +141,12 @@ falsification_tests: - id: FALSIFY-EG2L-007 rule: Generative verbs refuse by kind on every host prediction: apr chat and apr serve on a gemma-embedding2 GGUF fail with ModelLoadFailed (exit 6) naming apr embed, before a tokenizer or backend is built and with no GPU; generative and unparsed files pass through to the loaders - test: falsify_eg2l_007_* + test: falsify_eg2l_007_ if_fails: The refusal ran only inside the CUDA preload, so the CPU chat path ended an empty session with exit 0 - id: FALSIFY-EG2L-008 rule: SPM-style BPE ids equal the reference tokenizer prediction: Merges apply by text, lowest rank first and leftmost on ties; stale pairs are skipped; a space becomes U+2581 with no prefix; words split at newline runs; unknown symbols become <0xXX> tokens; specials split out first; a gemma4 file without merges is refused by name - test: gguf::spm_bpe::tests::* + test: gguf::spm_bpe::tests if_fails: The gemma4 vocabulary fell back to greedy longest-match, or a merge order, space or byte rule departs from llama.cpp; scripts/tokenizer_parity.sh reports the first id mismatch on a real file qa_gate: id: F-EG2L-001 From 1e225d393663a7cb4b8cb13e7e8a724b2176d29c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 18:28:43 +0200 Subject: [PATCH 17/21] chore(contracts): regenerate the ONT witness for the EG contract ids pv-sat on the merged tree: the census id set now includes the embeddinggemma2 load and forward contracts. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- ...1afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 37ac683405..b4a0aa53d8 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "7f990b6136d839df6dff4839a4190fa2452596855768f0ae36144ba44f9343e7", + "census_id_set_sha256": "9f925fae829b96df7534ec766908e03f759a13a6f2196cdf00013d362c30fdcf", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "27b283ae0515bdce9c9584bb4191071aa130fe12", + "reasoner_git_sha": null, "checkable_n": 84, "result": { "kind": "model", @@ -10,5 +10,5 @@ } }, "pc_reasoner": "fired", - "cpu_ms": 8 + "cpu_ms": 0 } From 54f74030abfdc35d968f4782e74cddab725762f5 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 11 Oct 2026 18:49:16 +0200 Subject: [PATCH 18/21] fix(contracts): serve refuses the embedding kind; bind four EG-2 forward tests Review of the EG-0..EG-2 branch found two contract statements that did not match the code. - apr-cli-commands-v1: `serve` declared `embedding: receipt`, but `apr serve run` refuses an embedding file with exit 6 and names `apr embed` (FALSIFY-EG2L-007). Under I-3 that cell is a refusal. It is now `refuse` with `use_instead: embed`; EG-3 (serve) flips it back when it lands. - embeddinggemma2-forward-v1: FALSIFY-EG2-001, 002, 007 and 008 named prose from the EG-0 pre-registration. EG-2 has a test for each, so each now cites it by name and the strict test-binding gate resolves it. 003, 004, 005, 006, 009 and 010 have no exact test yet and keep their prose. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- contracts/apr-cli-commands-v1.yaml | 3 ++- contracts/embeddinggemma2-forward-v1.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index e577ba2a4d..d5de32785b 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -64,7 +64,8 @@ commands: category: inference description: "Inference server (plan/run)" requires_model: true - kinds: {generative: receipt, embedding: receipt} + kinds: {generative: receipt, embedding: refuse} + use_instead: {embedding: embed} side_effects: [network, filesystem] subcommands: [plan, run] diff --git a/contracts/embeddinggemma2-forward-v1.yaml b/contracts/embeddinggemma2-forward-v1.yaml index c41ed6d960..e4e6e98977 100644 --- a/contracts/embeddinggemma2-forward-v1.yaml +++ b/contracts/embeddinggemma2-forward-v1.yaml @@ -159,12 +159,12 @@ falsification_tests: - id: FALSIFY-EG2-001 rule: Layer kind from the file prediction: A synthetic file whose global and local blocks are swapped yields swapped layer kinds - test: unit test over a synthetic tensor-shape table, no layer-index arithmetic in the classifier + test: falsify_eg2l_003_swapped_shapes_swap_kinds if_fails: Layer kind derived from an index pattern or an architecture constant (I-9) - id: FALSIFY-EG2-002 rule: Local attention window prediction: Attention weights beyond the window are exactly zero in local layers - test: unit test over n in [1, 3W] with random q, k and a V that marks each position; the boundary pair abs(i - j) = floor(W / 2) is inside, floor(W / 2) + 1 is outside + test: a_position_outside_the_window_has_no_effect_eg2 if_fails: Window off by one, the full W taken on each side instead of W / 2, or a causal mask applied instead of a symmetric one - id: FALSIFY-EG2-003 rule: Bidirectional global attention @@ -189,12 +189,12 @@ falsification_tests: - id: FALSIFY-EG2-007 rule: Unit norm and finite prediction: Every output vector has norm 1 within 1e-5 and no NaN or Inf - test: every corpus item at every declared dimension + test: embeds_to_a_unit_vector_of_the_output_width_eg2 if_fails: Missing normalisation, or a float16 activation overflow (I-5) - id: FALSIFY-EG2-008 rule: Truncation re-normalises the leading dims prediction: The 256 response equals the leading 256 of the 768 response, re-normalised - test: compare responses at 768, 512, 256 and 128 for every corpus item + test: truncation_keeps_the_prefix_direction_at_unit_length_eg2 if_fails: Truncation without re-normalisation, or the 512 response taken from the trunk - id: FALSIFY-EG2-009 rule: Batch and order invariance From 1f867389813fff0eb040023a95bebb9de0539402 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 10:28:32 +0200 Subject: [PATCH 19/21] chore(contracts): regenerate the ONT witness after the main merge pv-sat over the merged contracts/ tree. The contracts lib tests ran against this witness: 2292 passed, 0 failed. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- ...541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 7f0aa6a9fe..cbaf169959 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "ced96e20e1de8700f4de7592803f602f279ccecc086cbb4fbd86b349b9028779", + "census_id_set_sha256": "2691203479d4fee532ade8a5cf5d6cb57e110127f5ffe20edd4b3c7e912dfade", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "28006ac347c6c8ff337874e39711ea67dbfe8bcf", + "reasoner_git_sha": null, "checkable_n": 84, "result": { "kind": "model", From a931ca17158ac781a03b7c5c1891b56b1256e3c2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 11:12:57 +0200 Subject: [PATCH 20/21] fix(audit): re-audit the surface ledger rows whose cited lines this branch moved The branch edits extended_commands.rs and dispatch_analysis.rs, which 83 ledger rows cite by line, so the dogfood coverage gate (G2.1 freshness) was red. Every cited line was mapped through the branch diff and checked to carry the same text at its new number. Two rows already pointed at the wrong line on main and now cite the definition they describe: apr embed -> the Embed variant, apr dataset audio-inspect -> AudioInspect. check_dogfood_coverage.sh: PASS, self-test rc 0. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- docs/audits/surface_audit.csv | 168 +++++++++++++++++----------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index 9e537d7804..d5b3efb979 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -82,16 +82,16 @@ apr,POST /v1/gpu/warmup,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender apr,POST /v1/logprobs,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:222,low apr,POST /v1/perplexity,7,nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:223,low apr,POST /v1/predict,5,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:144,high -apr,apr attn-parity-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1007,high -apr,apr attn-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1021,high -apr,apr audio-inspect-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:984,high -apr,apr awq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:878,high +apr,apr attn-parity-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1028,high +apr,apr attn-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1042,high +apr,apr audio-inspect-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1005,high +apr,apr awq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:899,high apr,apr beat-run,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:429,high -apr,apr bench,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:88,high +apr,apr bench,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:109,high apr,apr canary check,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:45,medium apr,apr canary create,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:31,medium -apr,apr capability,6,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:655,high -apr,apr cbtop,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:478,high +apr,apr capability,6,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:676,high +apr,apr cbtop,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:499,high apr,apr cgp baseline,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:117,low apr,apr cgp bench,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:46,low apr,apr cgp compete,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:128,low @@ -117,16 +117,16 @@ apr,apr cgp profile wgpu,6,x86_64-linux,NONE,no,9,profiling,crates/aprender-cgp/ apr,apr cgp roofline,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:64,low apr,apr cgp trace,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:99,low apr,apr cgp tui,6,x86_64-linux,NONE,no,7,qa-cgp,crates/aprender-cgp/src/cli.rs:115,low -apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:40 (ACCELERATOR_GUARD crates/apr-cli/src/dispatch_analysis.rs:1710),high -apr,apr chat --backend cpu,6,x86_64-linux,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high -apr,apr chat --backend cuda,6,x86_64-linux;nvidia-cuda,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium -apr,apr chat --backend wgpu,6,UNKNOWN,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high +apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:61 (ACCELERATOR_GUARD crates/apr-cli/src/dispatch_analysis.rs:1710),high +apr,apr chat --backend cpu,6,x86_64-linux,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:106 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high +apr,apr chat --backend cuda,6,x86_64-linux;nvidia-cuda,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:106 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium +apr,apr chat --backend wgpu,6,UNKNOWN,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:106 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr check,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:781,high -apr,apr check-finite-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1041,low +apr,apr check-finite-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1062,low apr,apr code,2,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:836,high apr,apr code --input-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:891 (CodeInputFormat crates/apr-cli/src/commands_enum.rs:120),high apr,apr code --output-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:797 (CodeOutputFormat crates/apr-cli/src/commands_enum.rs:31),high -apr,apr compare-hf,9,x86_64-linux,safetensors,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:563,medium +apr,apr compare-hf,9,x86_64-linux,safetensors,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:584,medium apr,apr compile,9,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:682,medium apr,apr convert,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:604,high apr,apr data audit,3,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:54,high @@ -169,34 +169,34 @@ apr,apr data x registry show-info,6,x86_64-linux,huggingface-transformers,no,4,d apr,apr data x repl,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:175,low apr,apr data x schema,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:66,low apr,apr data x view,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:140,low -apr,apr dataset audio-inspect,6,x86_64-linux,huggingface-transformers,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1441,low -apr,apr ddp-metrics-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:953,high +apr,apr dataset audio-inspect,6,x86_64-linux,huggingface-transformers,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1511,low +apr,apr ddp-metrics-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:974,high apr,apr debug,3,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:292,high apr,apr debug embed-viz,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:978,low apr,apr decrypt,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:140,medium -apr,apr devices,8,x86_64-linux;nvidia-cuda,ollama,yes,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:647,high -apr,apr diagnose,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:834,medium +apr,apr devices,8,x86_64-linux;nvidia-cuda,ollama,yes,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:668,high +apr,apr diagnose,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:855,medium apr,apr diff,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:313,high apr,apr distill,9,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:143,medium -apr,apr dry-sampling-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:872,high -apr,apr embed,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1358,medium -apr,apr embed-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1064,low -apr,apr embeddings-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1173,high +apr,apr dry-sampling-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:893,high +apr,apr embed,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1413,medium +apr,apr embed-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1085,low +apr,apr embeddings-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1194,high apr,apr encrypt,4,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:121,high -apr,apr eval,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:124,high -apr,apr eval --task classify,6,x86_64-linux,scikit-learn,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1406 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task code,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1420 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task compare,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1444 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task contamination,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1441 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task correlation,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1446 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task human,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1447 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task humaneval,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1423 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1432 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1448 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1445 (flag crates/apr-cli/src/extended_commands.rs:145),high -apr,apr experiment view,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1568,low +apr,apr eval,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:145,high +apr,apr eval --task classify,6,x86_64-linux,scikit-learn,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1406 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task code,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1420 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task compare,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1444 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task contamination,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1441 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task correlation,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1446 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task human,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1447 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task humaneval,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1423 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1432 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1448 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1445 (flag crates/apr-cli/src/extended_commands.rs:166),high +apr,apr experiment view,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1613,low apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:447,high -apr,apr explain-token-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1066,low +apr,apr explain-token-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1087,low apr,apr export,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:476,high apr,apr export --format coreml,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high apr,apr export --format gguf,6,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high @@ -206,53 +206,53 @@ apr,apr export --format openvino,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-co apr,apr export --format safetensors,6,x86_64-linux;nvidia-cuda,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high apr,apr finetune,2,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:6,high apr,apr finetune --task classify,2,x86_64-linux,unsloth,no,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:48,high -apr,apr flow,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:668,high -apr,apr fp8-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:884,high -apr,apr gbnf-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1131,high -apr,apr gptq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:896,high +apr,apr flow,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:689,high +apr,apr fp8-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:905,high +apr,apr gbnf-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1152,high +apr,apr gptq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:917,high apr,apr gpu,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:829,high -apr,apr gpu-memtrace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1102,low -apr,apr grad-norm,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1167,high -apr,apr hang-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:935,high -apr,apr hex,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:588,high -apr,apr imatrix-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1167,high +apr,apr gpu-memtrace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1123,low +apr,apr grad-norm,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1188,high +apr,apr hang-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:956,high +apr,apr hex,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:609,high +apr,apr imatrix-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1188,high apr,apr import,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:506,high apr,apr inspect,3,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:224,high -apr,apr kernel parity,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1472,low -apr,apr kv-timeline-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1085,high +apr,apr kernel parity,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1517,low +apr,apr kv-timeline-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1106,high apr,apr lint,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:414,high apr,apr list,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:596,medium apr,apr manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:439,medium apr,apr mcp,1,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:773,high apr,apr merge,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:709,high -apr,apr modelfile parse,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1590,low -apr,apr monitor,9,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:458,medium +apr,apr modelfile parse,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1635,low +apr,apr monitor,9,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:479,medium apr,apr mono archive,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:39,high apr,apr mono audit,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:36,high apr,apr mono publish,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:22,high apr,apr mono shims,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:29,high -apr,apr nccl-diag-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:911,high -apr,apr nf4-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:890,high -apr,apr ollama-chat-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:849,high -apr,apr ollama-tools-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:858,high -apr,apr oom-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:902,low +apr,apr nccl-diag-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:932,high +apr,apr nf4-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:911,high +apr,apr ollama-chat-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:870,high +apr,apr ollama-tools-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:879,high +apr,apr oom-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:923,low apr,apr oracle,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:84,medium -apr,apr otlp-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1108,high -apr,apr parity,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:317,medium -apr,apr parity --per-op,9,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1516),high -apr,apr parity --out,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:333 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high -apr,apr parity --threshold,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:336 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high +apr,apr otlp-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1129,high +apr,apr parity,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:338,medium +apr,apr parity --per-op,9,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:351 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1516),high +apr,apr parity --out,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:354 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high +apr,apr parity --threshold,7,"x86_64-linux,aarch64-linux",NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:357 (dispatch crates/apr-cli/src/dispatch_analysis.rs:1517),high apr,apr pipeline apply,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:28,low apr,apr pipeline plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:9,low apr,apr pipeline status,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:51,low apr,apr pipeline validate,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/pipeline_commands.rs:58,low -apr,apr ppl,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1198,medium -apr,apr pretrain,6,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:720,high -apr,apr profile,3,x86_64-linux;nvidia-cuda,llama.cpp,yes,9,profiling,crates/apr-cli/src/extended_commands.rs:165,high -apr,apr prometheus-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1113,high +apr,apr ppl,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1219,medium +apr,apr pretrain,6,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:741,high +apr,apr profile,3,x86_64-linux;nvidia-cuda,llama.cpp,yes,9,profiling,crates/apr-cli/src/extended_commands.rs:186,high +apr,apr prometheus-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1134,high apr,apr prune,4,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:113,high -apr,apr ptx,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:362,high -apr,apr ptx-map,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330,high +apr,apr ptx,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:383,high +apr,apr ptx-map,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:351,high apr,apr publish,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:46,high apr,apr pull,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:542,high apr,apr pv audit,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:67,low @@ -353,9 +353,9 @@ pv,pv validate,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-c pv,pv verify-bindings,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:452,low pv,pv verify-pipeline,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:445,low pv,pv verify-structure,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:434,low -apr,apr qa,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:238,high -apr,apr qualify,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:686,high -apr,apr quant-preservation-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1215,high +apr,apr qa,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:259,high +apr,apr qualify,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:707,high +apr,apr quant-preservation-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1236,high apr,apr quantize,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:742,high apr,apr quantize --scheme fp16,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high apr,apr quantize --scheme int4,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high @@ -374,12 +374,12 @@ apr,apr rag index,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/a apr,apr rag info,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:307,low apr,apr rag query,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:184,low apr,apr rag transcribe,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:228,low -apr,apr react-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:923,low +apr,apr react-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:944,low apr,apr registry aliases,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/registry.rs:12,high -apr,apr registry-quota-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1161,high -apr,apr rerank,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1253,high +apr,apr registry-quota-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1182,high +apr,apr rerank,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1274,high apr,apr rm,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:599,medium -apr,apr rm-gc-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1186,high +apr,apr rm-gc-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1207,high apr,apr rosetta chain,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:70,low apr,apr rosetta compare-inference,2,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:109,high apr,apr rosetta convert,4,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:43,high @@ -401,9 +401,9 @@ apr,apr run --trace-level basic,2,x86_64-linux;nvidia-cuda,NONE,no,10,apr-core-c apr,apr run --trace-level chrome,2,x86_64-linux;nvidia-cuda,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:110 (TRACE_LEVEL_VALUES crates/apr-cli/src/commands_enum.rs:23),high apr,apr run --trace-level layer,2,x86_64-linux;nvidia-cuda,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:110 (TRACE_LEVEL_VALUES crates/apr-cli/src/commands_enum.rs:23),high apr,apr run --trace-level payload,2,x86_64-linux;nvidia-cuda,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:110 (TRACE_LEVEL_VALUES crates/apr-cli/src/commands_enum.rs:23),high -apr,apr runs diff,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1544,low -apr,apr runs ls,4,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1511,high -apr,apr runs show,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1529,low +apr,apr runs diff,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1589,low +apr,apr runs ls,4,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1556,high +apr,apr runs show,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1574,low apr,apr serve plan,9,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:17,medium apr,apr serve run,4,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:38,high apr,apr serve run --backend cpu,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:73 (NO value_parser; consumed crates/apr-cli/src/commands/serve/types.rs:57),high @@ -411,8 +411,8 @@ apr,apr serve run --backend cuda,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr- apr,apr serve run --backend wgpu,4,UNKNOWN,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:73 (NO value_parser; consumed crates/apr-cli/src/commands/serve/types.rs:57),high apr,apr serve run --no-cors,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:53 (router.rs:33-42),high apr,apr serve run --no-metrics,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/serve_commands.rs:53 (router.rs:33-42),high -apr,apr shard,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1214,high -apr,apr shared-cache-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1192,high +apr,apr shard,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1235,high +apr,apr shared-cache-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1213,high apr,apr showcase,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:5,high apr,apr sim emc-check,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:105,low apr,apr sim emc-validate,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:110,low @@ -425,9 +425,9 @@ apr,apr sim verify,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate apr,apr sim version,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:119,low apr,apr stamp,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:628,high apr,apr tensors,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:352,high -apr,apr test llm bench,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1696 (dispatch crates/apr-cli/src/commands/test_llm.rs:482),medium -apr,apr test llm bench --band,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1831 (protocol crates/apr-cli/src/commands/test_llm_band.rs:348),medium -apr,apr test tensor,4,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1648,high +apr,apr test llm bench,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1741 (dispatch crates/apr-cli/src/commands/test_llm.rs:482),medium +apr,apr test llm bench --band,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1876 (protocol crates/apr-cli/src/commands/test_llm_band.rs:348),medium +apr,apr test tensor,4,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1693,high apr,apr tokenize apply,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:38,low apr,apr tokenize encode,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:125,high apr,apr tokenize encode-corpus,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:149,low @@ -435,7 +435,7 @@ apr,apr tokenize import-hf,6,x86_64-linux,huggingface-transformers,no,10,apr-cor apr,apr tokenize plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:14,low apr,apr tokenize repair-manifest,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:239,low apr,apr tokenize train,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:63,low -apr,apr tool-use-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1125,high +apr,apr tool-use-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1146,high apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:370,high apr,apr train apply,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:76,high apr,apr train apply --task classify,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:88,high @@ -449,12 +449,12 @@ apr,apr train plan --task pretrain,3,x86_64-linux,pytorch,no,10,apr-core-command apr,apr train submit,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:284,low apr,apr train sweep,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:201,high apr,apr train watch,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:172,low -apr,apr tree,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:666,high +apr,apr tree,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:687,high apr,apr tui,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:810,medium -apr,apr tune,4,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:384,high -apr,apr typical-p-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1137,high -apr,apr unified-search-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1190,high -apr,apr unshard,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1229,medium +apr,apr tune,4,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:405,high +apr,apr typical-p-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1158,high +apr,apr unified-search-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1211,high +apr,apr unshard,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1250,medium apr,apr validate,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:280,high apr,apr validate-manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:299,medium apr,apr zram benchmark,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:47,low From 84578e78baf0ddc4bba5ac83a3fea7c91496f46e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 12:14:00 +0200 Subject: [PATCH 21/21] fix(embed): refuse '..' in the embed script paths; rustfmt matmul_fused.rs Two guard-cargo rows were red on the PR head: - bashrs SEC/DET/IDEM (#3196): five SEC010 path-traversal findings in apr_embed_corpus.sh and apr_embed_probe.sh. Both now refuse an output path containing '..' (exit 2), and the probe refuses a verb that is not a plain name before it becomes a directory. check_bashrs_gate.sh: PASS, 551 files, 0 SEC/DET/IDEM errors. - include!d files formatted (#4151): matmul_fused.rs, touched by EG-2, was not rustfmt-clean. Formatted; include_fmt_ratchet.sh rc 0. Refs #4901 Agent: apr-embed Co-Authored-By: Claude Opus 5.5 --- crates/aprender-serve/src/gguf/inference/matmul_fused.rs | 4 +++- scripts/apr_embed_corpus.sh | 1 + scripts/apr_embed_probe.sh | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs index a441479d42..ef979a01fd 100644 --- a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs +++ b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs @@ -178,7 +178,9 @@ impl OwnedQuantizedModel { // arm, the IQ*/Q2_K/Q3_K dequant fallback (PMAT-3477 / #3091) let data = &weight.data; match weight.qtype { - GGUF_TYPE_F32 => Ok(Self::fused_matmul_f32(input, data, in_dim, out_dim, seq_len)), + GGUF_TYPE_F32 => Ok(Self::fused_matmul_f32( + input, data, in_dim, out_dim, seq_len, + )), GGUF_TYPE_BF16 => Ok(float16_matmul( input, data, diff --git a/scripts/apr_embed_corpus.sh b/scripts/apr_embed_corpus.sh index b91ca21057..a7a094c958 100755 --- a/scripts/apr_embed_corpus.sh +++ b/scripts/apr_embed_corpus.sh @@ -10,6 +10,7 @@ set -euo pipefail OUT="${2:-docs/audits/APR-EMBED-001/corpus}" +case "${OUT}" in *..*) echo "apr_embed_corpus: outdir may not contain '..': ${OUT}" >&2; exit 2 ;; esac if [ -n "${1:-}" ]; then COMMIT="$1" elif [ -f "${OUT}/SOURCE" ]; then diff --git a/scripts/apr_embed_probe.sh b/scripts/apr_embed_probe.sh index 0dc2c8f355..e19c259164 100755 --- a/scripts/apr_embed_probe.sh +++ b/scripts/apr_embed_probe.sh @@ -10,6 +10,7 @@ set -euo pipefail MODEL="${1:?model file}" BF16="${2:?bf16 sibling}" OUT="${3:?output tsv}" +case "${OUT}" in *..*) echo "apr_embed_probe: output path may not contain '..': ${OUT}" >&2; exit 2 ;; esac TABLE="docs/audits/APR-EMBED-001/EG-0/probe-verbs.tsv" TIMEOUT_S="${APR_EMBED_PROBE_TIMEOUT:-120}" @@ -27,6 +28,7 @@ printf 'verb\texit\tfirst_line\n' >> "${OUT}" n=0 while IFS=$'\t' read -r verb args; do case "${verb}" in ''|'#'*) continue ;; esac + case "${verb}" in *..*|*/*) echo "apr_embed_probe: verb is not a plain name: ${verb}" >&2; exit 2 ;; esac o="${SCRATCH}/${verb}" mkdir -p "${o}" args="${args//\{M\}/${MODEL}}"