diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92fa61de8b..8a7f4dd716 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1211,6 +1211,14 @@ jobs: run: bash scripts/derive_model_manifest.sh --check - name: C14 model-parity case table (L0-1a, #2971) run: bash scripts/check_model_parity.sh --self-test + # #3577: the parity-receipt denominator is the SECOND, independent reader of the + # receipt corpus — the extractor is the first, and they share no code so a defect + # in either is caught by the other rather than by nobody. It shipped unwired, which + # `check_guards_are_wired.sh` caught: a refusal nothing runs is not a refusal, and + # `parity-receipt-v1.yaml`'s own equations name "both readers are wired into CI" as + # a precondition. Its 4-case self-test is cargo-free and model-free, so it runs here. + - name: Parity-receipt denominator case table (#3577) + run: bash scripts/parity_receipt_denominator.sh --self-test # F-1 (#3022, #3024): a model file the user NAMED is the model that answers, or # the command refuses — `apr chat` silently loaded its toy demo model for a # `model.safetensors.index.json` (exit 0, zero tokens, the real path in the banner) diff --git a/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd b/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd new file mode 100644 index 0000000000..b294234cdd --- /dev/null +++ b/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts diff --git a/contracts/census.json b/contracts/census.json index 016e5b245e..f2f05ea3be 100644 --- a/contracts/census.json +++ b/contracts/census.json @@ -1,8 +1,8 @@ { "schema": "ont.paiml.dev/census/v1alpha1", "git_sha": null, - "n_files": 1801, - "n_parsed": 1801, + "n_files": 1803, + "n_parsed": 1803, "n_parse_errors": 0, "parse_errors": [], "quarantined_n": 0, @@ -12,7 +12,7 @@ "kernel": 363, "model-family": 28, "model-family-variant": 1, - "pattern": 87, + "pattern": 89, "pretraining-corpus": 2, "registry": 519, "schema": 766, @@ -22,14 +22,15 @@ }, "by_entity_type": { "gguf": 2, + "json": 2, "pv-contract": 1 }, "by_anchoring": { "unanchored": 1798, - "class": 2, + "class": 4, "instance": 1 }, - "id_set_sha256": "e624814d2c82e8a53ab24faac0b5281c0dad15618d9f14e970c740c8391cb712", + "id_set_sha256": "717a1cd852f535664eb9d0d7c90ef6e8c2666362c992264f025fc86031e7de02", "declared_external": [ { "name": "provable-contracts", diff --git a/contracts/contracts.nt b/contracts/contracts.nt index dc9a62ac7d..b2c784a8b4 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -8411,6 +8411,29 @@ . "contracts/entrenar/parity-profiling-system-v1.yaml"^^ . "parity-profiling-system-v1"^^ . + . + . + . + "parity-receipt"^^ . + "contracts/parity-receipt-v1.yaml"^^ . + "parity-receipt-v1"^^ . + "pattern"^^ . + "parity-receipt-v1"^^ . + "superseded"^^ . + "1.0.0"^^ . + . + . + . + . + . + . + "parity-receipt"^^ . + "contracts/parity-receipt-v2.yaml"^^ . + "parity-receipt-v2"^^ . + "pattern"^^ . + "parity-receipt-v2"^^ . + "active"^^ . + "2.0.0"^^ . . . "contracts/bashrs/parser-soundness-v1.yaml"^^ . @@ -9409,6 +9432,113 @@ . "false"^^ . "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785"^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-08"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-08"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.66.0"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "noah-Lambda-Vector"^^ . + "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-06"^^ . + "noah-Lambda-Vector"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-06"^^ . + "noah-Lambda-Vector"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + . . "cpu=ok"^^ . "cuda=ok"^^ . diff --git a/contracts/ontology.yaml b/contracts/ontology.yaml index e8ad447c68..e026c513a5 100644 --- a/contracts/ontology.yaml +++ b/contracts/ontology.yaml @@ -140,6 +140,10 @@ entity_types: - {name: apr-model, extractor: apr_model, implemented: true} - {name: gguf, extractor: gguf, implemented: true} - {name: csv, extractor: csv, implemented: false} + # ONT-4c3 (aprender#3577): the logit-parity receipts under evidence/parity/**. Subclasses `json` for now; + # it moves under a shared `Receipt` class once quorum and dispatch receipts join, so the three families + # inherit common shapes. NOT the throughput parity receipts — a different family with a different validator. + - {name: parity-receipt, extractor: parity_receipt, implemented: true} extractors: - {name: pv_contract, reader: ontology/extract/pv_contract.rs, implemented: true} @@ -151,6 +155,7 @@ extractors: - {name: apr_model, reader: ontology/extract/apr_model.rs, implemented: true} - {name: gguf, reader: ontology/extract/gguf.rs, implemented: true} - {name: csv, reader: ontology/extract/csv.rs, implemented: false} + - {name: parity_receipt, reader: ontology/extract/parity_receipt.rs, implemented: true} # What this ontology does NOT express, said out loud so nobody encodes it by accident. Each names the reader that # would have to change first. diff --git a/contracts/parity-receipt-v1.yaml b/contracts/parity-receipt-v1.yaml new file mode 100644 index 0000000000..4ca904c25a --- /dev/null +++ b/contracts/parity-receipt-v1.yaml @@ -0,0 +1,120 @@ +# ────────────────────────────────────────────── +# parity-receipt-v1 — the RETIRED logit-parity layout (ONT-001 §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHAT THIS IS. The layout the seven logit-parity records carried from 2026-09-06 until #3577 migrated them: +# the bare `apr parity --json` document, with the readings and the judgment mixed at one level. +# +# {"model": "./m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ … ]} +# +# It is recorded rather than deleted because a version nobody wrote down is a version somebody re-invents. The +# fork was real and cost real time: an extractor keyed to one layout silently skips the other, and skipping is +# indistinguishable from passing — which is how a record can sit in the tree for months with no comparator and +# no shape able to say so. +# +# STATUS: superseded by parity-receipt-v2, WITH NO INSTANCES IN THE TREE. All seven records were migrated to v2 +# in the same commit as the comparator back-fill (#3577), so this contract declares no shape: a shape whose +# target class nothing instantiates passes vacuously, and a vacuous pass is worth less than no shape at all. +# The enforcement that replaces it is stronger and lives where it can fire — `extract:parity-receipt` REFUSES +# an unmigrated record BY NAME (a file with no v2 `schema` but a top-level `metrics[]` or `parity`), and +# `scripts/parity_receipt_denominator.sh` refuses it independently. A returning v1 record is an error naming the +# file, not a silent skip. +# +# WHAT V2 CHANGED, and why each: +# · `schema` — v1 had none, so nothing could tell a parity record from any other JSON in the directory. +# · raw readings separated from the judgment (`raw` vs `result`) — the #3574 layout. `raw` is the original +# document byte for byte, so the migration is a relabel and the measurement is never restated. +# · `comparator {kind, comparator_sha, reason}` — the field whose absence this row exists to make impossible. +# All seven were self-comparisons (apr-CPU vs apr-CUDA) that implied an oracle they never had. +# · `partially_receipted`, `unmeasured[]` — an empty completeness claim now has to be written deliberately. +# · `host`, `backend`, `apr_version`, `generated_at`, `threshold_source` — recovered from committed evidence +# (each record's `provenance.record` names the RECORD.md it was quoted from), never invented. +# · `cell.model_sha256` — carried ONLY by the one record that recorded it at measurement time. The other six +# say so in `unmeasured` instead of being back-filled from a hash taken today. +# +# KIND: pattern (historical). No shape, by construction — see STATUS. +# ────────────────────────────────────────────── +name: parity-receipt-v1 +version: "1.0.0" +scope: > + The retired logit-parity record layout, recorded so the fork it caused is documented and a returning instance + is recognisable. Out of scope: everything live — the shapes, the extractor and the denominator are + parity-receipt-v2's. +status: superseded + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The pre-#3577 logit-parity layout: no schema, readings and judgment at one level, no comparator. Superseded + by parity-receipt-v2, with no instances in the tree; an unmigrated instance is refused by name by the + extractor and by the committed denominator predicate rather than by a shape that would pass vacuously. + references: + - 'aprender#3577 (the migration and the back-fill), #3576, #3269 (ONT-4c3)' + - 'contracts/parity-receipt-v2.yaml — the live contract' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — refuses an unmigrated record by name' + - 'scripts/parity_receipt_denominator.sh — refuses one independently' + - 'evidence/parity/l0-1/{lambda,gx10}/RECORD.md, evidence/parity/l0-1b/gx10/n5/DETERMINISM.md — the provenance the back-fill quotes' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-sigma-v1] + +equations: + # A `kind: pattern` contract declares no SHAPE — see STATUS, a shape over a class nothing + # instantiates passes vacuously — but it still ASSERTS things, and those are statable. These two + # are the invariants below written as the equations the corpus integrity check reads, not new + # claims invented to satisfy a counter: PRC1-INV-001 and PRC1-INV-002 said exactly this already. + no_instances: + formula: "legacy(f) ⇔ f ∈ evidence/parity/**/*.json ∧ ¬f.schema ∧ (f.metrics[] ∨ f.parity)" + domain: every *.json under evidence/parity/, walked in byte order + codomain: the empty set — all seven records were migrated to v2 in the #3577 commit + invariants: + - "the retired layout is recognised by ABSENCE of `schema` plus PRESENCE of a top-level + reading: either half alone is some other document and is not claimed here" + - a count of zero is asserted, not assumed — the denominator predicate measures it independently + preconditions: + - the seven records named in evidence/parity/*/RECORD.md have been migrated + postconditions: + - a v1 record reappearing under evidence/parity/ makes this equation false and is visible + lean_theorem: "none — L4 not declared" + refused_by_name: + formula: "legacy(f) ⇒ f ∈ errors(extract:parity-receipt) ∧ exit(parity_receipt_denominator.sh) ≠ 0" + domain: a planted v1-layout record + codomain: two independent refusals, each naming the file + invariants: + - "refused BY NAME, never skipped: a skip is indistinguishable from a pass, which is how a + record sat in the tree for months with no comparator and nothing able to say so" + - "two readers, not one: the extractor and the shell predicate share no code, so a defect in + either is caught by the other rather than by nobody" + preconditions: + - both readers are wired into CI — a refusal nothing runs is not a refusal + postconditions: + - removing the planted record makes both pass again, so the pair can return to green + lean_theorem: "none — L4 not declared" + +invariants: + - id: PRC1-INV-001 + property: the retired layout has no instances in the committed tree + formal: '∀ f ∈ evidence/parity/**/*.json: schema(f) = "apr-parity-receipt/v2" ∨ ¬legacy(f)' + prose: false + - id: PRC1-INV-002 + property: a returning instance is refused by name, by two independent readers + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors(extract:parity-receipt) ∧ exit(denominator.sh) ≠ 0' + prose: false + +falsification_tests: + - id: FALSIFY-PRC1-001 + rule: no instance survives + prediction: > + planting a v1-layout record under evidence/parity/ makes both the extractor and the denominator predicate + refuse it by name, and removing it makes both pass again — both directions + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt && bash scripts/parity_receipt_denominator.sh --self-test + if_fails: the retired layout returns and nothing says so diff --git a/contracts/parity-receipt-v2.yaml b/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/contracts/shapes.ttl b/contracts/shapes.ttl index 06b73b15c7..7c45d68c6b 100644 --- a/contracts/shapes.ttl +++ b/contracts/shapes.ttl @@ -112,3 +112,109 @@ ] ; . + a sh:NodeShape ; + sh:targetClass ; + sh:closed true ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "cpu" "cuda" "wgpu" "metal" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 1 ; + sh:pattern "^[0-9a-f]{64}$" ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "llama_cpp" "transformers" "self" ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; +. + diff --git a/crates/apr-cli/src/commands/parity_admission.rs b/crates/apr-cli/src/commands/parity_admission.rs index 1b585e9094..370e30b101 100644 --- a/crates/apr-cli/src/commands/parity_admission.rs +++ b/crates/apr-cli/src/commands/parity_admission.rs @@ -387,7 +387,15 @@ mod sentinel_tests { fn record(rel: &str) -> (usize, f32) { let text = std::fs::read_to_string(evidence(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")); let v: serde_json::Value = serde_json::from_str(&text).expect("valid JSON"); - let rows = v["metrics"].as_array().expect("metrics is an array"); + // A v2 receipt (#3577) embeds the raw `apr parity --json` document under `raw`, so the + // readings live at `raw.metrics`; a fresh run carries them at the top level. One rule, + // stated once: look inside the envelope when there is one. + let doc = if v.get("raw").is_some_and(serde_json::Value::is_object) { + &v["raw"] + } else { + &v + }; + let rows = doc["metrics"].as_array().expect("metrics is an array"); let cos: Vec = rows .iter() .filter_map(|r| r["cosine_similarity"].as_f64()) diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index 4cf16a329c..6048ee9303 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -287,6 +287,23 @@ fn decide_shapes_gate( reason: Reason::NoFocus, } .into()), + ShapesOutcome::WrongCorpus { + shapes_n, + expected, + found, + refused, + } => { + eprintln!( + "shapes: extract:parity-receipt matched {found} focus node(s); evidence/parity/EXPECTED_RECEIPTS says {expected} ({shapes_n} shape(s))" + ); + for r in &refused { + eprintln!("shapes: refused {r}"); + } + Err(LintDeclined { + reason: Reason::WrongCorpus, + } + .into()) + } ShapesOutcome::NoReceipts { shapes_n, dir } => { // ONT-4c1: the WHY travels with the decline — the lattice has no ReceiptUnmeasured element (ONT-6's // 15 reasons), so the reason is NoCheckable and this line says what could not be checked. diff --git a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs new file mode 100644 index 0000000000..ef4f4cde02 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs @@ -0,0 +1,220 @@ +//! ONT-4c3 (PMAT-3577) — logit-parity receipts as focus nodes, on the CLI. +//! +//! The three answers `pv lint --gate shapes` may give about this family, each on its own fixture, so that a +//! build which collapses any two of them fails here: +//! +//! | fixture | exit | why | +//! |---|---|---| +//! | `parity-green` | 0 | a complete v2 receipt: one focus node, no violation | +//! | `parity-nocomparator` | 1 | the state all seven records were in before #3577 back-filled them | +//! | `parity-unknownkind` | 1 | a comparator kind the shape does not accept — this is what the `sh:in` MUTATION breaks | +//! | `parity-unmigrated` | 2 | a legacy record refused BY NAME: `Unknown{WrongCorpus}`, never Pass | +//! | `parity-denominator-drift` | 2 | 2 receipts, denominator 1 — a receipt added without bumping the count | +//! +//! DISCRIMINATION, in both directions. `parity-green` must PASS, so a build that declines every parity corpus +//! fails this file; `parity-nocomparator` and `parity-unknownkind` must FAIL, so a build whose shapes accept +//! anything fails it too. A table of only-invalid cases is passed by a validator that rejects everything. +//! +//! THE MUTATION. Widening `in: [llama_cpp, transformers, self]` to accept `oracle` makes +//! `parity-unknownkind` pass, and this file goes RED. The fixture copies of the contract are asserted +//! byte-identical to `contracts/parity-receipt-v2.yaml`, so mutating the real contract without the fixtures is +//! caught by the drift test instead — either way, RED. A mutation that no committed case can see is not a +//! control. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn fixture(name: &str) -> PathBuf { + repo_root().join("tests/fixtures/ont").join(name) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +impl Run { + fn all(&self) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + self.code, self.stdout, self.stderr + ) + } +} + +fn shapes_on(name: &str) -> Run { + let contracts = fixture(name).join("contracts"); + let out = Command::new(pv_bin()) + .args([ + "lint", + contracts.to_str().expect("utf-8 path"), + "--gate", + "shapes", + "--format", + "json", + ]) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn violations(r: &Run) -> usize { + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + v["extra"]["violations"].as_u64().unwrap_or(0) as usize +} + +fn focus_nodes(r: &Run) -> usize { + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + v["extra"]["focus_nodes_n"].as_u64().unwrap_or(0) as usize +} + +#[test] +fn a_complete_receipt_passes_with_one_focus_node() { + let r = shapes_on("parity-green"); + assert_eq!(r.code, 0, "{}", r.all()); + assert_eq!(violations(&r), 0, "{}", r.all()); + assert_eq!(focus_nodes(&r), 1, "{}", r.all()); + // The shapes are ARMED in the fixture (no lint-baseline.json narrows them), so a violation here would be + // a real Fail rather than a reported one — which is what makes the two FAIL cases below meaningful. + assert!(r.stdout.contains("parity-receipt-complete"), "{}", r.all()); +} + +#[test] +fn a_receipt_with_no_comparator_fails_naming_the_property() { + // The state every one of the seven records was in before #3577: a self-comparison implying an oracle. + let r = shapes_on("parity-nocomparator"); + assert_eq!(r.code, 1, "{}", r.all()); + assert_eq!(violations(&r), 1, "exactly one violation\n{}", r.all()); + assert!(r.stdout.contains("parity/comparator"), "{}", r.all()); + assert!(r.stdout.contains("minCount"), "{}", r.all()); +} + +#[test] +fn a_comparator_kind_the_shape_does_not_accept_fails() { + // THE MUTATION TARGET. Widen `in:` to accept `oracle` and this case goes green — so this assertion is + // what makes that mutation detectable. + let r = shapes_on("parity-unknownkind"); + assert_eq!(r.code, 1, "{}", r.all()); + assert!(violations(&r) >= 1, "{}", r.all()); + assert!( + r.stdout.contains("llama_cpp") || r.stdout.contains("\"in\"") || r.stdout.contains("(in)"), + "the report must name the `in` constraint that rejected the kind\n{}", + r.all() + ); +} + +#[test] +fn an_unmigrated_legacy_record_declines_with_exit_2_and_is_named() { + let r = shapes_on("parity-unmigrated"); + assert_eq!(r.code, 2, "a refusal is a DECLINE, not a Fail\n{}", r.all()); + assert!(r.stderr.contains("UNMIGRATED"), "{}", r.all()); + assert!( + r.stderr.contains("legacy.json"), + "named by file\n{}", + r.all() + ); +} + +#[test] +fn a_receipt_added_without_bumping_the_denominator_declines_naming_both_numbers() { + // The falsifier the row names: the extractor found 2, the committed denominator says 1. + let r = shapes_on("parity-denominator-drift"); + assert_eq!(r.code, 2, "{}", r.all()); + assert!(r.stderr.contains("matched 2"), "{}", r.all()); + assert!(r.stderr.contains("says 1"), "{}", r.all()); +} + +#[test] +fn the_three_answers_are_distinct() { + // A build that collapses decline into fail, or fail into pass, fails here rather than in a release. + let codes: Vec = ["parity-green", "parity-nocomparator", "parity-unmigrated"] + .iter() + .map(|f| shapes_on(f).code) + .collect(); + assert_eq!(codes, vec![0, 1, 2], "pass / fail / decline must differ"); +} + +#[test] +fn the_entity_type_is_counted_in_by_entity_type_not_merely_registered() { + // ONT-001 v4.10's probe asks `by_entity_type["parity-receipt"] == 7`. Registering the entity + // type in Σ is not enough for that: without a key here the probe reads ABSENT, and an absent + // key is not zero — a consumer treating it as one measures nothing and calls it a pass. Same + // shape as #3610, one map over. + let out = Command::new(pv_bin()) + .args(["lint", "contracts", "--gate", "shapes", "--format", "json"]) + .current_dir(repo_root()) + .output() + .expect("failed to spawn pv"); + let v: serde_json::Value = + serde_json::from_str(&String::from_utf8_lossy(&out.stdout)).expect("json report"); + let counted = &v["extra"]["by_entity_type"]["parity-receipt"]; + assert!( + !counted.is_null(), + "by_entity_type carries no `parity-receipt` key — the probe would read ABSENT, not 7" + ); + assert_eq!( + counted.as_u64(), + v["extra"]["by_shape"] + .as_array() + .expect("by_shape") + .iter() + .find_map(|s| s + .as_str()? + .strip_prefix("parity-receipt-complete=")? + .parse() + .ok()), + "the entity count and the shape's focus-node count must be the same number" + ); +} + +#[test] +fn every_fixture_carries_the_real_contract_byte_for_byte() { + // A fixture copy that drifts from `contracts/parity-receipt-v2.yaml` would let the real shape be mutated + // while the case table stayed green — the mutation control's blind spot, closed here. + let real = std::fs::read(repo_root().join("contracts/parity-receipt-v2.yaml")) + .expect("the real contract is in the tree"); + for name in [ + "parity-green", + "parity-nocomparator", + "parity-unknownkind", + "parity-unmigrated", + "parity-denominator-drift", + ] { + let copy = std::fs::read(fixture(name).join("contracts/parity-receipt-v2.yaml")) + .unwrap_or_else(|e| panic!("{name} carries the contract: {e}")); + assert_eq!( + copy, real, + "{name}'s copy of parity-receipt-v2.yaml has drifted from contracts/" + ); + } +} + +#[test] +fn the_committed_tree_agrees_with_its_own_denominator() { + // The real corpus, not a fixture: the independent predicate and the extractor must reach the same count. + let out = Command::new("bash") + .arg("scripts/parity_receipt_denominator.sh") + .current_dir(repo_root()) + .output() + .expect("failed to spawn the denominator predicate"); + assert!( + out.status.success(), + "exit {:?}\n{}{}", + out.status.code(), + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); +} diff --git a/crates/aprender-contracts/src/lint/mod.rs b/crates/aprender-contracts/src/lint/mod.rs index 47f2950b46..db3b347077 100644 --- a/crates/aprender-contracts/src/lint/mod.rs +++ b/crates/aprender-contracts/src/lint/mod.rs @@ -737,6 +737,10 @@ fn shapes_result(contract_dir: &Path, validation_passed: bool) -> (GateResult, V skipped_gate("shapes", &format!("{shapes_n} shape(s), no focus node")), Vec::new(), ), + shapes_gate::ShapesOutcome::WrongCorpus { shapes_n, expected, found, refused } => ( + skipped_gate("shapes", &format!("extract:parity-receipt matched {found} focus node(s) and evidence/parity/EXPECTED_RECEIPTS says {expected} ({shapes_n} shape(s)){} — an extractor that saw the wrong corpus reports the same \"no violations\" as one that saw all of it", if refused.is_empty() { String::new() } else { format!("; refused: {}", refused.join("; ")) })), + Vec::new(), + ), shapes_gate::ShapesOutcome::NoReceipts { shapes_n, dir } => ( skipped_gate("shapes", &format!("{shapes_n} shape(s) resolve receipts and the tree holds none under {dir}/ — R-2: unmeasured is a decline")), Vec::new(), diff --git a/crates/aprender-contracts/src/lint/shapes_gate.rs b/crates/aprender-contracts/src/lint/shapes_gate.rs index e813127d18..1a82eac3fc 100644 --- a/crates/aprender-contracts/src/lint/shapes_gate.rs +++ b/crates/aprender-contracts/src/lint/shapes_gate.rs @@ -59,6 +59,16 @@ pub enum ShapesOutcome { NoFocus { shapes_n: usize }, /// A shape resolves receipts and the tree holds none under `evidence/dogfood/models/`. NoReceipts { shapes_n: usize, dir: String }, + /// PMAT-3577 — `extract:parity-receipt` matched a different number of focus nodes than + /// `evidence/parity/EXPECTED_RECEIPTS` says the tree holds, or it refused a record by name. An + /// extractor that silently sees the wrong corpus reports the same "no violations" as one that sees + /// all of it, so this is `Unknown{WrongCorpus}` — never `Pass`, never a fabricated `Fail`. + WrongCorpus { + shapes_n: usize, + expected: usize, + found: usize, + refused: Vec, + }, /// A positive control did not fire. PositiveControlFailed { shapes_n: usize, @@ -146,6 +156,36 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { Ok(x) => x, Err(e) => return ShapesOutcome::ExtractFailed(e), }; + // PMAT-3577: the count is pinned before anything is graded. A miss here is not a corpus verdict. + if let Some((expected, found)) = extraction.parity.wrong_corpus() { + return ShapesOutcome::WrongCorpus { + shapes_n: shapes.len(), + expected, + found, + refused: extraction + .parity + .errors + .iter() + .map(ToString::to_string) + .collect(), + }; + } + if !extraction.parity.errors.is_empty() { + return ShapesOutcome::WrongCorpus { + shapes_n: shapes.len(), + expected: extraction + .parity + .expected + .unwrap_or(extraction.parity.records), + found: extraction.parity.records, + refused: extraction + .parity + .errors + .iter() + .map(ToString::to_string) + .collect(), + }; + } let graph = extraction.graph; let needs_receipts = shapes.iter().any(|s| { s.properties @@ -230,6 +270,11 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { extraction.gguf.rungs.len() + extraction.gguf.files_read, ), ("apr-model", extraction.apr_model.files_read), + // ONT-4c3: registered in Σ and implemented, so it is counted here like every other entity + // type. Without this key a probe asking `by_entity_type["parity-receipt"]` reads ABSENT — + // and an absent key is not zero, so a consumer that treats it as one measures nothing and + // calls it a pass. The same shape as #3610, one map over. + ("parity-receipt", extraction.parity.records), ("code", extraction.code.symbols), ("lean", extraction.lean.statements), ] diff --git a/crates/aprender-contracts/src/ontology/extract/mod.rs b/crates/aprender-contracts/src/ontology/extract/mod.rs index 053e8f0d1e..3f2c54939c 100644 --- a/crates/aprender-contracts/src/ontology/extract/mod.rs +++ b/crates/aprender-contracts/src/ontology/extract/mod.rs @@ -19,6 +19,7 @@ pub mod code; pub mod gguf; pub mod json; pub mod lean; +pub mod parity_receipt; pub mod pv_contract; /// Every extractor's output over `contract_dir`, plus the input-side warnings the extractors chose to carry @@ -41,6 +42,8 @@ pub struct Extraction { pub code: code::CodeStats, /// ONT-4b2: the in-tree Lean theorems and the contracts that cite them. pub lean: lean::LeanStats, + /// ONT-4c3: the logit-parity receipts under `evidence/parity/**`, and the files this extractor refused. + pub parity: parity_receipt::ParityStats, } /// What a walk could not do. Every variant is the DECLARATION's fault (exit 3), never a corpus verdict. @@ -87,6 +90,7 @@ pub fn all(contract_dir: &Path) -> Result { out.resolve = receipts::resolve(&mut out.graph, &out.gguf.rungs, &out.receipts); out.code = code::extract(contract_dir, &mut out.graph); out.lean = lean::extract(contract_dir, &mut out.graph); + out.parity = parity_receipt::extract(root, &mut out.graph); Ok(out) } diff --git a/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs b/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs new file mode 100644 index 0000000000..1a9c0968a0 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs @@ -0,0 +1,303 @@ +//! ONT-001 §3.7, §5 ONT-4c3 — `extract:parity-receipt`: a logit-parity record under +//! `evidence/parity/**` becomes a `parity:ParityReceipt` focus node (PMAT-3577, aprender#3577). +//! +//! **Which family this is.** Two artifact families in this tree share the word "parity". This module reads the +//! **logit** family — `apr parity --json`, CPU vs CUDA, a cosine per position — and nothing else. The +//! **throughput** family (apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 cross-class +//! defect) is validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py --parity`, +//! which this row deliberately does NOT touch: its fixtures require `instrument`, `protocol_ref` and +//! `lanes`, none of which a logit record has ever carried. One validator per artifact family — folding one +//! into the other would delete coverage from whichever family nobody was watching. +//! +//! **Why this exists at all.** Until this row, the logit records had NO validator of any kind. That is why +//! seven of them sat in the tree with no comparator for months and nothing noticed: there was nothing that +//! could have noticed. +//! +//! **Three outcomes per file, and skipping is never one of them silently.** Every `*.json` under +//! `evidence/parity/**` is one of: +//! +//! | class | rule | consequence | +//! |---|---|---| +//! | record | `schema` == [`SCHEMA`] | a focus node | +//! | **unmigrated** | no v2 schema, but a top-level `metrics[]` or `parity` | **refused BY NAME** — never skipped | +//! | other | neither | skipped, counted in [`ParityStats::skipped`] | +//! +//! The middle row is the point. A file that looks like a parity record and carries no schema is the legacy +//! layout this row migrated away; leaving one behind would be a record the graph cannot see, and an extractor +//! that silently skips is indistinguishable from one that passes. +//! +//! **The count is pinned, because an extractor that matches nothing reports the same "no violations" as one +//! that matches everything.** [`EXPECTED_FILE`] holds the expected focus-node count, produced by the committed +//! predicate `scripts/parity_receipt_denominator.sh`. A mismatch is `Unknown{WrongCorpus}`, exit 2 — never +//! `Pass`, never a fabricated `Fail`. + +use std::path::{Path, PathBuf}; + +use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; +use crate::ontology::shapes::RDFS_SUBCLASS_OF; + +/// The one layout this reader accepts. The legacy layout has no `schema` key at all and is refused by name. +pub const SCHEMA: &str = "apr-parity-receipt/v2"; +/// Where the logit-parity records live, relative to the repository root. +pub const EVIDENCE_DIR: &str = "evidence/parity"; +/// The committed denominator: the focus-node count the extractor must reproduce. +pub const EXPECTED_FILE: &str = "evidence/parity/EXPECTED_RECEIPTS"; + +/// The vocabulary root for parity receipts: `https://ont.paiml.dev/v1alpha1/parity/`. +#[must_use] +pub fn parity(name: &str) -> String { + format!("{}parity/{name}", crate::ontology::rdf::ONT_BASE) +} + +/// A record this reader refuses, by name. A refusal is never a corpus verdict — it is this file's fault. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParityError { + pub file: String, + pub what: String, +} + +impl std::fmt::Display for ParityError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.file, self.what) + } +} + +impl std::error::Error for ParityError {} + +/// What the walk found. `records` is what the denominator pins. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ParityStats { + /// Files carrying [`SCHEMA`] — one focus node each. + pub records: usize, + /// Files under the tree that are not parity records at all (`props-*`, `derived_expiries`, …). + pub skipped: usize, + /// Records whose `threshold_source` names a path that is not in the tree. + pub threshold_source_missing: usize, + /// Files refused by name: an unmigrated legacy record, or a record this reader cannot read. + pub errors: Vec, + /// The committed expectation, when [`EXPECTED_FILE`] is present and parses. + pub expected: Option, +} + +impl ParityStats { + /// The extractor matched a different number of records than the committed denominator says. + /// + /// `None` when the denominator is absent — that is a different fault (a missing declaration), reported + /// by the caller, and deliberately not folded in here: "no expectation" and "a broken expectation" are + /// not the same state. + #[must_use] + pub fn wrong_corpus(&self) -> Option<(usize, usize)> { + match self.expected { + Some(n) if n != self.records => Some((n, self.records)), + _ => None, + } + } +} + +/// Read the committed denominator: the first non-comment, non-blank line, parsed as a count. +fn read_expected(root: &Path) -> Option { + let text = std::fs::read_to_string(root.join(EXPECTED_FILE)).ok()?; + text.lines() + .map(str::trim) + .find(|l| !l.is_empty() && !l.starts_with('#')) + .and_then(|l| l.parse().ok()) +} + +/// A document that carries no v2 `schema` but has the legacy layout's fingerprint: a top-level `metrics` +/// array, or a top-level `parity` key. Such a file is an unmigrated record, not an unrelated document. +fn looks_legacy(v: &serde_json::Value) -> bool { + v.get("metrics").is_some_and(serde_json::Value::is_array) || v.get("parity").is_some() +} + +/// Every `*.json` under `/evidence/parity/**`, in byte order, classified and — for records — emitted. +pub fn extract(root: &Path, g: &mut Graph) -> ParityStats { + let mut stats = ParityStats { + expected: read_expected(root), + ..ParityStats::default() + }; + let mut files = Vec::new(); + walk(&root.join(EVIDENCE_DIR), &mut files); + files.sort(); + if !files.is_empty() { + declare_classes(g); + } + for f in files { + let rel = f + .strip_prefix(root) + .unwrap_or(&f) + .to_string_lossy() + .replace('\\', "/"); + let Ok(text) = std::fs::read_to_string(&f) else { + stats.errors.push(ParityError { + file: rel, + what: "unreadable".into(), + }); + continue; + }; + let Ok(v) = serde_json::from_str::(&text) else { + // Not JSON at all under a tree of JSON: say so rather than counting it as "other". + stats.errors.push(ParityError { + file: rel, + what: "not JSON".into(), + }); + continue; + }; + match v.get("schema").and_then(serde_json::Value::as_str) { + Some(SCHEMA) => { + emit(g, root, &rel, &v, &mut stats); + stats.records += 1; + } + other if looks_legacy(&v) => { + stats.errors.push(ParityError { + file: rel, + what: format!( + "an UNMIGRATED logit-parity record: schema {} is not {SCHEMA}. The legacy layout \ + was migrated by #3577; a record the extractor cannot see is a record no shape \ + can refuse.", + other.map_or_else(|| "absent".to_string(), |s| format!("{s:?}")) + ), + }); + } + _ => stats.skipped += 1, + } + } + stats +} + +fn walk(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + walk(&p, out); + } else if p.extension().and_then(|x| x.to_str()) == Some("json") { + out.push(p); + } + } +} + +/// `SelfComparedReceipt` and `OracleComparedReceipt` are subclasses of `ParityReceipt`, so a shape targeting +/// the parent sees every receipt while the two child shapes carry the constraints that differ by comparator +/// kind — the subset has no `sh:or`, and two shapes over one class is how that split is expressed. +fn declare_classes(g: &mut Graph) { + for child in ["SelfComparedReceipt", "OracleComparedReceipt"] { + g.insert( + parity(child), + RDFS_SUBCLASS_OF.to_string(), + Term::iri(parity("ParityReceipt")), + ); + } +} + +fn s(v: &serde_json::Value, k: &str) -> Option { + v.get(k).and_then(|x| match x { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Null => None, + other => Some(other.to_string()), + }) +} + +/// One record → one focus node. Every property the shapes name is written here; nothing else is, so +/// `closed: true` with an empty `ignoredProperties` is a statement about this function. +fn emit(g: &mut Graph, root: &Path, rel: &str, v: &serde_json::Value, stats: &mut ParityStats) { + let node = iri("parity-receipt", rel); + let comparator = v.get("comparator"); + let kind = comparator.and_then(|c| s(c, "kind")).unwrap_or_default(); + let class = if kind == "self" { + "SelfComparedReceipt" + } else { + "OracleComparedReceipt" + }; + g.insert(node.clone(), RDF_TYPE, Term::iri(parity(class))); + g.insert(node.clone(), parity("file"), Term::string(rel)); + for (key, prop) in [ + ("host", "host"), + ("backend", "backend"), + ("apr_version", "aprVersion"), + ("generated_at", "generatedAt"), + ("threshold_source", "thresholdSource"), + ] { + if let Some(val) = s(v, key) { + g.insert(node.clone(), parity(prop), Term::string(val)); + } + } + if let Some(b) = v + .get("partially_receipted") + .and_then(serde_json::Value::as_bool) + { + g.insert(node.clone(), parity("partiallyReceipted"), Term::boolean(b)); + } + if let Some(sha) = v.get("cell").and_then(|c| s(c, "model_sha256")) { + g.insert(node.clone(), parity("modelSha256"), Term::string(sha)); + } + for u in v + .get("unmeasured") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + { + g.insert(node.clone(), parity("unmeasured"), Term::string(u)); + } + // `resolves:` is ours, not SHACL: the extractor does the resolving and materialises the failure as a + // literal a shape can refuse with `maxCount 0`. The THRESHOLD VALUE is never read here — a threshold + // typed into a shape instead of resolved from thresholds.yaml is this row's STOP condition. + if let Some(src) = s(v, "threshold_source") { + if !root.join(&src).exists() { + stats.threshold_source_missing += 1; + g.insert( + node.clone(), + parity("thresholdSourceMissing"), + Term::string(src), + ); + } + } + if let Some(c) = comparator { + let cnode = iri("parity-comparator", rel); + g.insert(cnode.clone(), RDF_TYPE, Term::iri(parity("Comparator"))); + g.insert(cnode.clone(), parity("kind"), Term::string(&kind)); + if let Some(sha) = s(c, "comparator_sha") { + g.insert(cnode.clone(), parity("comparatorSha"), Term::string(sha)); + } + if let Some(r) = s(c, "reason") { + g.insert(cnode.clone(), parity("reason"), Term::string(r)); + } + g.insert(node, parity("comparator"), Term::iri(cnode)); + } +} + +/// The positive control (R-3): a copy of a real record with `comparator` removed must lose its comparator +/// edge, every run. Drawn beside the corpus so "the extractor still reads this layout" is measured rather +/// than assumed. +#[must_use] +pub fn positive_control(sample: &serde_json::Value) -> bool { + let mut g = Graph::new(); + let mut stats = ParityStats::default(); + let root = Path::new("."); + emit(&mut g, root, "__pc_sample__", sample, &mut stats); + let with = !g + .objects( + &iri("parity-receipt", "__pc_sample__"), + &parity("comparator"), + ) + .is_empty(); + let mut stripped = sample.clone(); + if let Some(o) = stripped.as_object_mut() { + o.remove("comparator"); + } + let mut g2 = Graph::new(); + emit(&mut g2, root, "__pc_planted__", &stripped, &mut stats); + let without = g2 + .objects( + &iri("parity-receipt", "__pc_planted__"), + &parity("comparator"), + ) + .is_empty(); + with && without +} + +#[cfg(test)] +#[path = "parity_receipt_tests.rs"] +mod tests; diff --git a/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs b/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs new file mode 100644 index 0000000000..4ea4a66f20 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs @@ -0,0 +1,230 @@ +//! PMAT-3577 — the extractor's case table. Every case is a state that either happened in this tree or is +//! one edit away from happening, and each says which. + +use super::*; + +fn record(kind: &str, extra: serde_json::Value) -> serde_json::Value { + let mut v = serde_json::json!({ + "schema": SCHEMA, + "cell": {"model": "m", "file": "./m.gguf", "quant": "Q4_K_M"}, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": {"kind": kind, "reason": "no oracle arm exists for this cell"}, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": ["ORACLE ARM: not measured."], + "result": {"positions": 78, "parity": true}, + "raw": {"model": "./m.gguf", "metrics": []} + }); + if let (Some(o), Some(e)) = (v.as_object_mut(), extra.as_object()) { + for (k, val) in e { + o.insert(k.clone(), val.clone()); + } + } + v +} + +fn emit_one(v: &serde_json::Value) -> (Graph, ParityStats) { + let mut g = Graph::new(); + let mut stats = ParityStats::default(); + emit(&mut g, Path::new("."), "r.json", v, &mut stats); + (g, stats) +} + +fn objects(g: &Graph, prop: &str) -> Vec { + g.objects(&iri("parity-receipt", "r.json"), &parity(prop)) + .iter() + .filter_map(|t| t.as_literal().map(|l| l.0.to_string())) + .collect() +} + +#[test] +fn a_self_compared_record_types_as_the_self_subclass_and_carries_every_named_property() { + let (g, _) = emit_one(&record("self", serde_json::json!({}))); + let node = iri("parity-receipt", "r.json"); + assert_eq!( + g.objects(&node, RDF_TYPE)[0].as_iri(), + Some(parity("SelfComparedReceipt").as_str()) + ); + assert_eq!(objects(&g, "host"), ["gx10-a5b5"]); + assert_eq!(objects(&g, "backend"), ["cuda"]); + assert_eq!(objects(&g, "aprVersion"), ["0.65.2"]); + assert_eq!(objects(&g, "generatedAt"), ["2026-09-08"]); + assert_eq!(objects(&g, "partiallyReceipted"), ["true"]); + assert_eq!(objects(&g, "unmeasured").len(), 1); +} + +#[test] +fn a_non_self_comparator_types_as_the_oracle_subclass() { + let v = record( + "llama_cpp", + serde_json::json!({"comparator": {"kind": "llama_cpp", "comparator_sha": "39173bcac"}}), + ); + let (g, _) = emit_one(&v); + assert_eq!( + g.objects(&iri("parity-receipt", "r.json"), RDF_TYPE)[0].as_iri(), + Some(parity("OracleComparedReceipt").as_str()) + ); + let c = iri("parity-comparator", "r.json"); + assert_eq!( + g.objects(&c, &parity("comparatorSha"))[0] + .as_literal() + .map(|l| l.0), + Some("39173bcac") + ); +} + +#[test] +fn model_sha256_is_absent_when_the_record_carries_none_and_absent_is_legal() { + // ONT-4c1's rule, and the reason the shape gives model_sha256 a pattern and no minCount: six of the + // seven back-filled records never recorded a model hash, and hashing the file on the host TODAY would + // attach a claim about a different world to a receipt about 2026-09-06. + let (g, _) = emit_one(&record("self", serde_json::json!({}))); + assert!(objects(&g, "modelSha256").is_empty()); + let with = record( + "self", + serde_json::json!({"cell": {"model_sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"}}), + ); + let (g2, _) = emit_one(&with); + assert_eq!(objects(&g2, "modelSha256").len(), 1); +} + +#[test] +fn a_threshold_source_that_names_no_file_is_materialised_for_a_shape_to_refuse() { + let v = record( + "self", + serde_json::json!({"threshold_source": "evidence/parity/does-not-exist.yaml"}), + ); + let (g, stats) = emit_one(&v); + assert_eq!(stats.threshold_source_missing, 1); + assert_eq!( + objects(&g, "thresholdSourceMissing"), + ["evidence/parity/does-not-exist.yaml"] + ); +} + +#[test] +fn the_plant_removes_the_comparator_edge_every_run() { + assert!(positive_control(&record("self", serde_json::json!({})))); +} + +#[test] +fn an_unmigrated_legacy_record_is_refused_by_name_and_never_skipped() { + // The exact layout the seven records carried before #3577: no schema, metrics[] at the top level. + let dir = tempdir("unmigrated"); + let f = dir.join(EVIDENCE_DIR).join("l0-1/lambda"); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write( + f.join("legacy.json"), + r#"{"model":"./m.gguf","tokens":78,"passed":78,"failed":0,"parity":true,"metrics":[]}"#, + ) + .expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.records, 0); + assert_eq!(stats.skipped, 0, "a legacy record must not be SKIPPED"); + assert_eq!(stats.errors.len(), 1); + assert!( + stats.errors[0].what.contains("UNMIGRATED"), + "{:?}", + stats.errors[0] + ); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn a_document_that_is_not_a_parity_record_is_skipped_and_counted() { + let dir = tempdir("other"); + let f = dir.join(EVIDENCE_DIR); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write(f.join("props-abc.json"), r#"{"seed": 1, "cases": []}"#).expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!( + (stats.records, stats.skipped, stats.errors.len()), + (0, 1, 0) + ); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn the_denominator_pins_the_count_and_a_mismatch_is_reported_in_both_directions() { + let dir = tempdir("denominator"); + let f = dir.join(EVIDENCE_DIR); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write( + f.join("r.json"), + serde_json::to_string(&record("self", serde_json::json!({}))).expect("json"), + ) + .expect("write"); + + // Committed 2, found 1 — a record was deleted or the extractor stopped seeing it. + std::fs::write(dir.join(EXPECTED_FILE), "# count\n2\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), Some((2, 1))); + + // Committed 1, found 1 — the state the gate requires. + std::fs::write(dir.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), None); + + // Committed 0, found 1 — a receipt added without updating the denominator. THE falsifier the row names. + std::fs::write(dir.join(EXPECTED_FILE), "0\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), Some((0, 1))); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn an_absent_denominator_is_not_a_mismatch_it_is_a_different_fault() { + // "no expectation" and "a broken expectation" must not collapse into one state: the first is a missing + // declaration for the caller to report, the second is WrongCorpus. + let stats = ParityStats { + records: 3, + expected: None, + ..ParityStats::default() + }; + assert_eq!(stats.wrong_corpus(), None); +} + +#[test] +fn pointing_the_extractor_at_a_subdirectory_trips_the_denominator() { + // The row's second falsifier: a narrowed walk finds fewer records and must not read as "no violations". + let dir = tempdir("subdir"); + let deep = dir.join(EVIDENCE_DIR).join("l0-1/lambda"); + std::fs::create_dir_all(&deep).expect("mkdir"); + std::fs::write( + deep.join("r.json"), + serde_json::to_string(&record("self", serde_json::json!({}))).expect("json"), + ) + .expect("write"); + std::fs::write(dir.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g = Graph::new(); + assert_eq!(extract(&dir, &mut g).wrong_corpus(), None); + + // Same denominator, a root whose evidence/parity holds nothing: 1 expected, 0 found. + let narrow = tempdir("subdir-narrow"); + std::fs::create_dir_all(narrow.join(EVIDENCE_DIR)).expect("mkdir"); + std::fs::write(narrow.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g2 = Graph::new(); + assert_eq!(extract(&narrow, &mut g2).wrong_corpus(), Some((1, 0))); + std::fs::remove_dir_all(&dir).ok(); + std::fs::remove_dir_all(&narrow).ok(); +} + +fn tempdir(tag: &str) -> PathBuf { + let p = std::env::temp_dir().join(format!( + "pmat3577-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&p).expect("tempdir"); + p +} diff --git a/crates/aprender-contracts/src/ontology/verdict.rs b/crates/aprender-contracts/src/ontology/verdict.rs index 614702615b..9e1a25db49 100644 --- a/crates/aprender-contracts/src/ontology/verdict.rs +++ b/crates/aprender-contracts/src/ontology/verdict.rs @@ -10,7 +10,7 @@ //! //! `meet = min` is Kleene's strong conjunction (K3). Exit mapping: Pass→0, Fail→1, Unknown→2 with a //! `decline: ` line. Proof obligations, all discharged by the `#[cfg(kani)]` harnesses below and -//! exhaustively by the unit tests over the 17 elements: +//! exhaustively by the unit tests over the 18 elements: //! //! - KANI-ONT-6-1 — meet laws: commutative, associative, idempotent, Pass is the identity, Fail absorbs, //! and the meet is below both operands. @@ -38,11 +38,22 @@ pub enum Reason { Advisory, ExtractorMissing, Prose, + /// PMAT-3577: an extractor RAN and read a different corpus than the tree declares — it matched a + /// different number of focus nodes than the committed denominator says, or refused a record by name. + /// Reading the wrong corpus reports the same "no violations" as reading all of it, so it is never + /// `Pass` and never a fabricated `Fail`. + /// + /// NOT named `ExtractorMiss`. [`Self::ExtractorMissing`] already means the opposite thing — an + /// extractor that does not exist — and the two would have sat one letter apart in the same lattice, + /// with the shorter a PREFIX of the longer: `grep ExtractorMiss` would match both, and any substring + /// test over the reasons would silently merge them. That is the defect class this repository keeps + /// paying for; the name says what happened instead. + WrongCorpus, } impl Reason { /// Every reason, in lattice order. - pub const ALL: [Self; 15] = [ + pub const ALL: [Self; 16] = [ Self::NotRun, Self::Skip, Self::Report, @@ -58,6 +69,7 @@ impl Reason { Self::Advisory, Self::ExtractorMissing, Self::Prose, + Self::WrongCorpus, ]; } @@ -77,7 +89,7 @@ pub enum Verdict { } impl Verdict { - /// The 17 elements: Fail, the 15 Unknowns in order, Pass. + /// The 18 elements: Fail, the 16 Unknowns in order, Pass. #[must_use] pub fn all() -> Vec { let mut v = vec![Self::Fail]; @@ -200,12 +212,12 @@ mod tests { use super::*; #[test] - fn seventeen_elements_in_lattice_order() { + fn eighteen_elements_in_lattice_order() { let all = Verdict::all(); - assert_eq!(all.len(), 17); + assert_eq!(all.len(), 18); assert!( all.windows(2).all(|w| w[0] < w[1]), - "Fail < Unknown(NotRun) < … < Unknown(Prose) < Pass" + "Fail < Unknown(NotRun) < … < Unknown(WrongCorpus) < Pass" ); } @@ -268,7 +280,7 @@ mod tests { assert_eq!(Verdict::Unknown(Reason::Skip).to_string(), "Unknown(Skip)"); let spellings: std::collections::HashSet = Verdict::all().iter().map(ToString::to_string).collect(); - assert_eq!(spellings.len(), 17, "no two elements share a spelling"); + assert_eq!(spellings.len(), 18, "no two elements share a spelling"); assert_eq!( serde_json::to_string(&Verdict::Unknown(Reason::NotArmed)) .ok() diff --git a/docs/audits/impl-PMAT-3577-receipt.md b/docs/audits/impl-PMAT-3577-receipt.md new file mode 100644 index 0000000000..73d4c6bcbb --- /dev/null +++ b/docs/audits/impl-PMAT-3577-receipt.md @@ -0,0 +1,120 @@ +# PMAT-3577 / #3577 — receipts under contract: `parity-receipt-v2` + `extract:parity-receipt` + the back-fill + +`checkout-only: true` — `pv` is 0.65.2 fleet-wide and has neither `lint --gate` nor `extract`, so every +verdict below was rendered in a checkout at the HEAD-built `pv` 0.68.2. A verdict on ≥ 1 fleet host is owed +once the pin lands (#3567), and this row does not wait for it. + +Worktree `/mnt/nvme-raid0/agent-wt/rel-3577`, branch `PMAT-3577-receipts-under-contract`, cut from +`origin/main` 863ba48ef. Measurements re-run by me, not taken from a lane. + +## What the row found before it built anything + +**1. The committed denominator is 7, not 8.** The ticket and the cop both said 8 (7 legacy + the #3574 +receipt). Measured over `git ls-files 'evidence/parity/**/*.json'`: 113 files, **7** parity records, **0** +with a `comparator`. The #3574 receipt is in PR #3575, which is **still open** — it is not on `main`. The +cop confirmed and will bump the denominator to 8 as part of landing #3575, which is this row's own +falsifier firing on its first real use. + +**2. Item 6 of the ticket is mis-aimed, in the way the ticket itself warns about.** The ruling said +"`receipt-lint` is deleted, replaced by `pv lint`"; the ticket corrected that (`receipt-lint` is +paiml-implement's quorum linter) and re-aimed item 6 at `scripts/check_parity_receipt.sh`. Measured: that +script validates a **different artifact family**. Its fixtures carry `instrument`, `protocol_ref`, `lanes[]` +with `decode_tok_per_sec` and a llama.cpp `build_commit`; `bench_receipt.py::validate_parity` requires +`instrument`, `protocol_ref`, `model`, `lanes`. A logit record has never carried one of them. Its callers +are `check_perf_claims_cite_receipts.sh`, `check_multiplatform_dogfood.sh` and `parity_host_receipt.sh`. +Folding it in would have deleted the validator for #2696 — published-apr-takes-the-CPU-path-and-reports-0.099x +— from a family nobody was watching. **`check_parity_receipt.sh` is out of scope and untouched**, agreed with +the cop, and the `done_when` line is satisfied by the honest answer: nothing to fold, different family. + +The ruling's premise gets *stronger*: "receipts are the only artifact family with no shape" is **literally +true** of the logit records. No validator of any kind, ever. That is why seven of them carried no comparator +for months — there was nothing that could have noticed. + +**3. `model_sha256` is `pattern` with no `minCount`, deliberately.** Six of the seven never recorded a model +hash. Hashing the files on the hosts today and attaching that to a receipt about 2026-09-06 would be a claim +about a different world wearing a witness's clothes. Absent means no measurement, never a match (ONT-4c1); +`partially_receipted: true` and an `unmeasured` entry carry the honesty. Confirmed by the cop before use. + +## What landed + +| | | +|---|---| +| `contracts/parity-receipt-v2.yaml` | 3 shapes: `parity-receipt-complete` (closed, `ignoredProperties: []`), `parity-comparator-self`, `parity-comparator-oracle` | +| `contracts/parity-receipt-v1.yaml` | the retired layout, recorded; **no shape** — see below | +| `crates/aprender-contracts/src/ontology/extract/parity_receipt.rs` | the extractor + 10 unit cases | +| `scripts/parity_receipt_denominator.sh` | the INDEPENDENT predicate + 4-case self-test | +| `evidence/parity/EXPECTED_RECEIPTS` | the committed denominator (7) | +| `crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs` | 8 CLI cases over 5 fixtures | +| `evidence/parity/**` (7 files) | migrated to v2 + back-filled, one commit | +| `contracts/{contracts.nt,shapes.ttl,census.json}` | regenerated (derived, R-18 / decision 8) | + +**v1 declares no shape, on purpose.** All seven instances were migrated, so a shape over the retired layout +would target a class nothing instantiates and pass vacuously — worth less than no shape. The enforcement that +replaces it fires: the extractor **refuses** an unmigrated record by name, and the denominator predicate +refuses it independently. + +**Nothing is armed.** Arming lives in `contracts/lint-baseline.json` `armed_shapes[]`, a shared file this row +is forbidden to touch (decision 7). The three shapes are computed and reported, exactly as `ladder-green` was +at ONT-4c1; arming them is a follow-up carrying the `touches-shared-contracts` label. + +## Controls, every one measured in both directions + +| control | before | after | +|---|---|---| +| **the back-fill** — comparator stripped from all 7 vs as committed | **7** parity-shape violations | **0** | +| **the plant** — comparator removed from one record | **exactly 1**, naming `ont:parity/comparator` and `minCount` | 0 on restore | +| **the mutation** — `in:` widened to accept `oracle`, contract + all 5 fixtures | `ont4c3_parity_receipts` **FAILS** (`a_comparator_kind_the_shape_does_not_accept_fails`) | 8/8 on restore | +| **the mutation, other half** — only the real contract widened | **FAILS** (`every_fixture_carries_the_real_contract_byte_for_byte`) | 8/8 on restore | +| **an unmigrated record** (hit for real when a `git checkout` reverted the migration mid-run) | `Unknown{WrongCorpus}`, **exit 2**, each legacy file refused by name | Pass after re-migration | +| **denominator drift** — 2 receipts, committed 1 | exit 2, "matched 2 … says 1" | — | +| **the three answers are distinct** | `parity-green` 0 · `parity-nocomparator` 1 · `parity-unmigrated` 2 | — | + +``` +cargo test -p aprender-contracts --lib ontology::extract::parity_receipt 10 passed +cargo test -p aprender-contracts --lib ontology::verdict 8 passed +cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts 8 passed +bash scripts/parity_receipt_denominator.sh --self-test 4 ok +bash scripts/parity_receipt_denominator.sh PASS 7 receipt(s), EXPECTED_RECEIPTS says 7 +pv lint contracts --gate shapes Pass, violations 0, parity-shape violations 0 +pv extract contracts --check rc 0 +``` + +`by_shape` on the committed tree: `parity-receipt-complete=7`, `parity-comparator-self=7`, +`parity-comparator-oracle=0`. + +## The back-fill is a relabel, and every field cites committed evidence + +`raw` in each migrated record is the original `apr parity --json` document, key for key. The envelope was +quoted from files already in the tree, named in each record's `provenance.record`: +`evidence/parity/l0-1/{lambda,gx10}/RECORD.md` and `evidence/parity/l0-1b/gx10/n5/DETERMINISM.md`. + +**One derived field was wrong on the first pass and is worth recording.** `result.verdict` initially copied +`raw.parity` — apr's own per-position band flag — which says PASS for the two 1.5B cells that their own +`RECORD.md` calls RED. It now resolves the threshold from `evidence/parity/thresholds.yaml` (`default.min_cosine` +0.98, `min_positions` 64) and judges min cosine over ≥ 64 positions, which reproduces all seven readings the +RECORD.md files state, including both REDs. A threshold is **never** typed into the shape — that is this row's +STOP condition — and `thresholdSource` is `resolves:`, with a missing path materialised as +`thresholdSourceMissing` for a shape to refuse with `maxCount 0`. + +Cross-check: the migrated `min_cosine` values reproduce `evidence/parity/thresholds.yaml`'s basis text to six +decimals (7B lambda 0.998607, 7B gx10 0.998465, 1.5B lambda 0.950827, 1.5B gx10 0.950611). + +## Not done, and why + +**`ONT-4c3 bound in the ONT-001 ledger` — cannot be done from this repository.** The ledger is +`docs/specifications/paiml-ontology.md` in **paiml/infra**, where v4.8 defines ONT-4c3 as **kernel** receipts +(`entity type kernel`, `apr-kernel-receipt/v1`, the `kernel-parity`/`kernel-timing`/`kernel-safety` shapes). +**Filed as paiml/infra#814** with both resolutions (renumber the parity row, or record the re-scope in a +v4.9) and no preference between them — the point is that one identifier should not mean two things in two +repositories. The re-scope is an aprender-side ruling that the infra spec does not yet carry, so binding +it is an **infra PR**, not this one. Raised with the cop rather than left as a checked box. The kernel +sub-row with the `gated_rmsnorm` fixture stays a follow-up either way. + +## Follow-ups + +- Arm the three shapes in `contracts/lint-baseline.json` (`touches-shared-contracts`, group of one). +- Bind ONT-4c3 in infra's ONT-001 ledger once **paiml/infra#814** decides which meaning the identifier keeps. +- #3575 bumps `EXPECTED_RECEIPTS` to 8 when it lands, and its receipt needs the v2 envelope + (`comparator`, `partially_receipted`, `backend`, `generated_at`) or these shapes will report it. +- `Receipt` as a shared parent class once quorum and dispatch receipts join (NOT this row). +- The dense oracle re-measurement, comparator ruling item (d). diff --git a/docs/audits/quorum-PMAT-3577.json b/docs/audits/quorum-PMAT-3577.json new file mode 100644 index 0000000000..33f19cdc89 --- /dev/null +++ b/docs/audits/quorum-PMAT-3577.json @@ -0,0 +1,288 @@ +{ + "ticket": "PMAT-3577", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "445224d9ed3509f8e938414933ad4948ffcce6d5", + "diff_sha256": "3eaaa279c9ff352034e9b3782a60e04be118337a00a02be37d76ef1d75ab6bac", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 683926, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly satisfies the requirements of PMAT-3577. It introduces the v2 shape for parity receipts (with thresholds correctly resolved via path rather than typed into the shape), fully skips adding a shape for v1 (in compliance with the 'do NOT add a v1 shape' STOP condition), updates the EXPECTED_RECEIPTS exactly to the back-fill denominator of 7, and explicitly documents why out-of-scope validations (e.g. check_parity_receipt.sh) are untouched. No weakened gates or unpermitted shared file modifications were found. The implementation strictly adheres to the stated ticket guidelines.", + "findings": [ + { + "claim": "The v1 receipt layout declares no shape, strictly abiding by the STOP condition 'do NOT add a v1 shape' as described in the amendment.", + "file": "contracts/parity-receipt-v1.yaml", + "grounding": "cited", + "line": 31 + }, + { + "claim": "The shape dynamically resolves the threshold via thresholdSource using 'resolves: path' instead of hardcoding any threshold value, satisfying the STOP condition.", + "file": "contracts/parity-receipt-v2.yaml", + "grounding": "cited", + "line": 55 + }, + { + "claim": "The denominator count for EXPECTED_RECEIPTS is correctly pinned to 7, exactly matching the number of back-filled receipts.", + "file": "evidence/parity/EXPECTED_RECEIPTS", + "grounding": "cited", + "line": 9 + } + ], + "raw_bytes": 4837, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff strictly implements the amended PMAT-3577 requirements. It introduces the parity-receipt-v2 shape, implements the extract:parity-receipt logic, successfully back-fills the 7 legacy receipts, and leaves the v1 contract intentionally shapeless as required by the amendment. The PR properly refrains from arming the shapes in `lint-baseline.json` and accurately updates only the in-scope `check_model_parity.sh`. All claims in the receipt accurately reflect the diff, tests pass successfully, and no gates are weakened (the `shapes_gate` is appropriately strengthened to decline on `WrongCorpus`).", + "findings": [ + { + "claim": "The diff correctly leaves parity-receipt-v1 shapeless in accordance with the amendment of PMAT-3577.", + "file": "contracts/parity-receipt-v1.yaml", + "grounding": "cited" + }, + { + "claim": "The denominator predicate correctly expects and matches 7 receipts.", + "command": "bash scripts/parity_receipt_denominator.sh", + "file": "evidence/parity/EXPECTED_RECEIPTS", + "grounding": "measured", + "output": "PASS 7 receipt(s), EXPECTED_RECEIPTS says 7" + }, + { + "claim": "The check_parity_receipt.sh script was left untouched.", + "command": "git diff --name-only origin/main...HEAD | grep check_parity_receipt.sh || echo \"Not touched\"", + "file": "scripts/check_parity_receipt.sh", + "grounding": "measured", + "output": "Not touched" + }, + { + "claim": "The gate shapes_gate was not weakened; it was actually strengthened to return WrongCorpus on a mismatch.", + "file": "crates/aprender-contracts/src/lint/shapes_gate.rs", + "grounding": "cited" + }, + { + "claim": "The tests for ont4c3 parity receipts pass, including the negative condition for a comparator kind the shape does not accept.", + "command": "cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts", + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "grounding": "measured", + "output": "running 9 tests\\ntest every_fixture_carries_the_real_contract_byte_for_byte ... ok\\ntest an_unmigrated_legacy_record_declines_with_exit_2_and_is_named ... ok\\ntest a_receipt_added_without_bumping_the_denominator_declines_naming_both_numbers ... ok\\ntest a_receipt_with_no_comparator_fails_naming_the_property ... ok\\ntest a_complete_receipt_passes_with_one_focus_node ... ok\\ntest a_comparator_kind_the_shape_does_not_accept_fails ... ok\\ntest the_three_answers_are_distinct ... ok\\ntest the_committed_tree_agrees_with_its_own_denominator ... ok\\ntest the_entity_type_is_counted_in_by_entity_type_not_merely_registered ... ok\\n\\ntest result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 8.13s" + } + ], + "raw_bytes": 8765, + "err_bytes": 59, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed ticket PMAT-3577, receipt PMAT-3577, refutation doctrine, and full diff in /mnt/nvme-raid0/agent-wt/rel-3577/docs/audits/quorum-PMAT-3577.prompt.md. Confirmed that parity-receipt-v2.yaml defines the 3 required shapes, parity-receipt-v1.yaml documents the superseded layout, extract:parity-receipt extracts logit-parity records into RDF focus nodes while refusing unmigrated records by name, the independent denominator predicate script passes all self-tests, all 7 legacy receipts under evidence/parity/** are back-filled to v2, and all unit and CLI integration tests pass cleanly without weakening any gates. Verdict: PASS.", + "findings": [ + { + "claim": "extract:parity-receipt unit tests pass cleanly with 10 passed tests, verifying classification, legacy record refusal by name, missing threshold detection, positive control, and denominator checks.", + "command": "cargo test -p aprender-contracts --lib ontology::extract::parity_receipt", + "file": "crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs", + "grounding": "measured", + "line": 1, + "output": "test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 1671 filtered out; finished in 0.01s" + }, + { + "claim": "All 9 CLI integration tests for ONT-4c3 parity receipts pass, verifying pass/fail/decline discrimination over 5 fixtures and ensuring contract byte-identical parity.", + "command": "cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts", + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "grounding": "measured", + "line": 1, + "output": "test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 7.97s" + }, + { + "claim": "The independent denominator predicate script passes its 4 self-test cases and confirms 7 receipts under evidence/parity/** match EXPECTED_RECEIPTS.", + "command": "bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 1, + "output": "ok a record is counted and an unrelated document is not (measured 1)\nok an unmigrated legacy record is refused by name\nok a receipt added without bumping the denominator disagrees\nok bumping the denominator makes them agree\nPASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." + }, + { + "claim": "Defines the active parity-receipt-v2 contract with 3 closed/specific shapes (parity-receipt-complete, parity-comparator-self, parity-comparator-oracle) and pins extractor reach via EXPECTED_RECEIPTS.", + "file": "contracts/parity-receipt-v2.yaml", + "grounding": "cited", + "line": 555 + } + ], + "raw_bytes": 10265, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "contracts/parity-receipt-v1.yaml", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The diff correctly leaves parity-receipt-v1 shapeless in accordance with the amendment of PMAT-3577." + ] + }, + { + "file": "contracts/parity-receipt-v1.yaml", + "line": 31, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The v1 receipt layout declares no shape, strictly abiding by the STOP condition 'do NOT add a v1 shape' as described in the amendment." + ] + }, + { + "file": "contracts/parity-receipt-v2.yaml", + "line": 55, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The shape dynamically resolves the threshold via thresholdSource using 'resolves: path' instead of hardcoding any threshold value, satisfying the STOP condition." + ] + }, + { + "file": "contracts/parity-receipt-v2.yaml", + "line": 555, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Defines the active parity-receipt-v2 contract with 3 closed/specific shapes (parity-receipt-complete, parity-comparator-self, parity-comparator-oracle) and pins extractor reach via EXPECTED_RECEIPTS." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The tests for ont4c3 parity receipts pass, including the negative condition for a comparator kind the shape does not accept." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "All 9 CLI integration tests for ONT-4c3 parity receipts pass, verifying pass/fail/decline discrimination over 5 fixtures and ensuring contract byte-identical parity." + ] + }, + { + "file": "crates/aprender-contracts/src/lint/shapes_gate.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The gate shapes_gate was not weakened; it was actually strengthened to return WrongCorpus on a mismatch." + ] + }, + { + "file": "crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "extract:parity-receipt unit tests pass cleanly with 10 passed tests, verifying classification, legacy record refusal by name, missing threshold detection, positive control, and denominator checks." + ] + }, + { + "file": "evidence/parity/EXPECTED_RECEIPTS", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The denominator predicate correctly expects and matches 7 receipts." + ] + }, + { + "file": "evidence/parity/EXPECTED_RECEIPTS", + "line": 9, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The denominator count for EXPECTED_RECEIPTS is correctly pinned to 7, exactly matching the number of back-filled receipts." + ] + }, + { + "file": "scripts/check_parity_receipt.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The check_parity_receipt.sh script was left untouched." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The independent denominator predicate script passes its 4 self-test cases and confirms 7 receipts under evidence/parity/** match EXPECTED_RECEIPTS." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (10265 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)" + ], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-3577.yaml b/docs/roadmaps/entries/PMAT-3577.yaml index 4ef5d6a57e..5fb0cb0987 100644 --- a/docs/roadmaps/entries/PMAT-3577.yaml +++ b/docs/roadmaps/entries/PMAT-3577.yaml @@ -14,4 +14,4 @@ estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered; pmat work add derived neither spec: nor acceptance_criteria: — defect paiml-mcp-agent-toolkit#1414). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). Done when: contracts/parity-receipt-v1.yaml with its shape on origin/main; extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; ONT-4c3 bound with the parity receipt as focus node; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml.' + notes: 'ACCEPTANCE (transcribed from gh#3577, incl. the dated AMENDMENT of 2026-09-21 02:00Z which supersedes the original item 1; §6 1e — the fragment transcribes the issue, it does not edit it). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). ITEM 1 AS AMENDED, verbatim: "contracts/parity-receipt-v2.yaml carries the shape (the constraints table below, unchanged) in its shape: block, exported to contracts/shapes.ttl. contracts/parity-receipt-v1.yaml is the SHAPELESS superseded form and says why in its own text: no v1 instance exists after the back-fill, and a shape over zero instances is a vacuous pass. A v1 record is refused BY NAME by the extractor (extract:parity-receipt) and, independently, by scripts/parity_receipt_denominator.sh — two readers, both wired (#3600 ci.yml step). A future v1 instance is therefore a refusal, not a silent pass." WHY AMENDED: the original item 1 said "parity-receipt-v1.yaml with its shape" and predates the migration — all 7 records went to v2 in the same commit, so v1 has ZERO instances, and a shape over a class nothing instantiates passes vacuously, which is the defect #3610 names and #3622 makes the shapes gate DECLINE on. Following it literally would add in one PR the thing the other PR exists to refuse. Both pro lanes read the original correctly and failed the PR on it; the ruling is on the acceptance, not the lanes. ONT-4c3 LEDGER BINDING: that ledger lives in paiml/infra (ONT-001 v4.10, infra#831) and CANNOT be done from this repository. The aprender-side deliverable is extractor + shape + denominator + receipts; the ledger row is infra s. The acceptance reads "bound in the ONT-001 ledger" as infra s row, CITED, not as a change here. REMAINING done_when, unchanged: extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml; and do NOT add a v1 shape — see the amendment.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 618b9a93ca..faf48904a6 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -18955,7 +18955,7 @@ roadmap: estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered; pmat work add derived neither spec: nor acceptance_criteria: — defect paiml-mcp-agent-toolkit#1414). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). Done when: contracts/parity-receipt-v1.yaml with its shape on origin/main; extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; ONT-4c3 bound with the parity receipt as focus node; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml.' + notes: 'ACCEPTANCE (transcribed from gh#3577, incl. the dated AMENDMENT of 2026-09-21 02:00Z which supersedes the original item 1; §6 1e — the fragment transcribes the issue, it does not edit it). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). ITEM 1 AS AMENDED, verbatim: "contracts/parity-receipt-v2.yaml carries the shape (the constraints table below, unchanged) in its shape: block, exported to contracts/shapes.ttl. contracts/parity-receipt-v1.yaml is the SHAPELESS superseded form and says why in its own text: no v1 instance exists after the back-fill, and a shape over zero instances is a vacuous pass. A v1 record is refused BY NAME by the extractor (extract:parity-receipt) and, independently, by scripts/parity_receipt_denominator.sh — two readers, both wired (#3600 ci.yml step). A future v1 instance is therefore a refusal, not a silent pass." WHY AMENDED: the original item 1 said "parity-receipt-v1.yaml with its shape" and predates the migration — all 7 records went to v2 in the same commit, so v1 has ZERO instances, and a shape over a class nothing instantiates passes vacuously, which is the defect #3610 names and #3622 makes the shapes gate DECLINE on. Following it literally would add in one PR the thing the other PR exists to refuse. Both pro lanes read the original correctly and failed the PR on it; the ruling is on the acceptance, not the lanes. ONT-4c3 LEDGER BINDING: that ledger lives in paiml/infra (ONT-001 v4.10, infra#831) and CANNOT be done from this repository. The aprender-side deliverable is extractor + shape + denominator + receipts; the ledger row is infra s. The acceptance reads "bound in the ONT-001 ledger" as infra s row, CITED, not as a change here. REMAINING done_when, unchanged: extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml; and do NOT add a v1 shape — see the amendment.' - id: PMAT-3587 github_issue: 3587 item_type: task diff --git a/evidence/parity/EXPECTED_RECEIPTS b/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..370d2c0701 --- /dev/null +++ b/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1,10 @@ +# PMAT-3577 / #3577 — the expected number of apr-parity-receipt/v2 focus nodes under evidence/parity/**. +# +# Produced by the committed predicate: bash scripts/parity_receipt_denominator.sh +# The extractor must reproduce this count exactly. A mismatch is Unknown{WrongCorpus}, exit 2 — never +# Pass, never a fabricated Fail. An extractor that silently matches fewer files than exist reports the same +# "no violations" as one that matches all of them, which is the vacuity hole this file closes. +# +# ADDING A RECEIPT? Update this number in the same commit. If you do not, the gate stops you — that is the +# falsifier working, not the gate being wrong. +7 diff --git a/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json index 19a6a0b884..dd79ff2c06 100644 --- a/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/gx10/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "21d182d69505159c", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1/gx10/RECORD.md heading '2026-09-08T12:5xZ' \u2014 hour stated, minute not", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9506108164787292, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "FAIL", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 15, - "gpu_argmax": 16, - "max_abs_diff": 12.008713722229004, - "mean_abs_diff": 1.4603080749511719, - "cosine_similarity": 0.9506108164787292, - "kl_divergence": 5.453976683901305, - "sigma_level": 2.138399453406798, - "cpk": 0.4525729853621727, - "verdict": "WarnOutOfSpec" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.4038071632385254, - "mean_abs_diff": 0.06531447172164917, - "cosine_similarity": 0.9998307228088379, - "kl_divergence": 0.00003169573582524447, - "sigma_level": 48.145552545660266, - "cpk": 15.786467404364531, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.47008025646209717, - "mean_abs_diff": 0.079141765832901, - "cosine_similarity": 0.9998854398727417, - "kl_divergence": 2.350304035683132e-6, - "sigma_level": 40.26894361164864, - "cpk": 13.157401595078596, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.34066465497016907, - "mean_abs_diff": 0.05701277032494545, - "cosine_similarity": 0.9998948574066162, - "kl_divergence": 0.00007930019129213315, - "sigma_level": 55.01750807501765, - "cpk": 18.0777776456949, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.36131715774536133, - "mean_abs_diff": 0.06654312461614609, - "cosine_similarity": 0.9998743534088135, - "kl_divergence": 5.586878639155519e-6, - "sigma_level": 48.151386331009974, - "cpk": 15.783450135247941, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.39881277084350586, - "mean_abs_diff": 0.05531281977891922, - "cosine_similarity": 0.9998670220375061, - "kl_divergence": 0.00012843890921633306, - "sigma_level": 56.67790489208867, - "cpk": 18.631383735800988, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.40886592864990234, - "mean_abs_diff": 0.06747263669967651, - "cosine_similarity": 0.9999094009399414, - "kl_divergence": 0.000011448242344448684, - "sigma_level": 47.16232087734927, - "cpk": 15.455593113910512, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.45634615421295166, - "mean_abs_diff": 0.06790852546691895, - "cosine_similarity": 0.9999128580093384, - "kl_divergence": 4.6069409166491056e-6, - "sigma_level": 46.829472258467334, - "cpk": 15.34481405203357, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3148663640022278, - "mean_abs_diff": 0.05503924563527107, - "cosine_similarity": 0.9998142719268799, - "kl_divergence": 0.0006964580484037847, - "sigma_level": 58.33280314138458, - "cpk": 19.17671825707048, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.37107133865356445, - "mean_abs_diff": 0.049693331122398376, - "cosine_similarity": 0.9998795390129089, - "kl_divergence": 0.0019487691332511425, - "sigma_level": 63.28510665986591, - "cpk": 20.832964906591553, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.3408195972442627, - "mean_abs_diff": 0.049113236367702484, - "cosine_similarity": 0.9999272227287292, - "kl_divergence": 0.0023197360102891846, - "sigma_level": 63.60800653625746, - "cpk": 20.942335923761355, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.2871994972229004, - "mean_abs_diff": 0.04618430510163307, - "cosine_similarity": 0.99989253282547, - "kl_divergence": 0.0010725620753604848, - "sigma_level": 67.48480073589947, - "cpk": 22.235205359724013, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.30663490295410156, - "mean_abs_diff": 0.05332659184932709, - "cosine_similarity": 0.9998827576637268, - "kl_divergence": 0.0003849714682721958, - "sigma_level": 60.14815823956778, - "cpk": 19.782094722778414, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.341217041015625, - "mean_abs_diff": 0.05129178240895271, - "cosine_similarity": 0.9998595714569092, - "kl_divergence": 0.0011121437754711468, - "sigma_level": 60.927599133008556, - "cpk": 20.048775947883673, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.3473668098449707, - "mean_abs_diff": 0.06534294039011002, - "cosine_similarity": 0.999890148639679, - "kl_divergence": 0.0018708287004877242, - "sigma_level": 50.02631084841449, - "cpk": 16.403031428829408, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3845231533050537, - "mean_abs_diff": 0.05668144300580025, - "cosine_similarity": 0.9998037815093994, - "kl_divergence": 0.0014187455405557119, - "sigma_level": 56.08514037244682, - "cpk": 18.43013123352451, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3365215063095093, - "mean_abs_diff": 0.051528360694646835, - "cosine_similarity": 0.999897837638855, - "kl_divergence": 0.001462163794542408, - "sigma_level": 61.80499140623471, - "cpk": 20.336271311252403, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.34671688079833984, - "mean_abs_diff": 0.05129155516624451, - "cosine_similarity": 0.999882161617279, - "kl_divergence": 0.00008450073376790388, - "sigma_level": 61.06263290998148, - "cpk": 20.09321118628562, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.3720208406448364, - "mean_abs_diff": 0.05188516154885292, - "cosine_similarity": 0.9998630881309509, - "kl_divergence": 0.0006878276376716025, - "sigma_level": 60.17461919193516, - "cpk": 19.79802557748553, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.364626407623291, - "mean_abs_diff": 0.077647864818573, - "cosine_similarity": 0.9997988343238831, - "kl_divergence": 0.004185557706680174, - "sigma_level": 43.092094054533995, - "cpk": 14.085197260353358, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.4366130828857422, - "mean_abs_diff": 0.0633498802781105, - "cosine_similarity": 0.9997689127922058, - "kl_divergence": 0.0032265442609676616, - "sigma_level": 49.67682516705651, - "cpk": 16.296689978441368, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 1.0816888809204102, - "mean_abs_diff": 0.16485518217086792, - "cosine_similarity": 0.9990662932395935, - "kl_divergence": 0.013121373568509257, - "sigma_level": 19.17278201676311, - "cpk": 6.1275329662463855, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.8328787088394165, - "mean_abs_diff": 0.12702322006225586, - "cosine_similarity": 0.9992517232894897, - "kl_divergence": 0.021413090149460762, - "sigma_level": 24.71860116353615, - "cpk": 7.977880694909801, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.8862452507019043, - "mean_abs_diff": 0.10550703853368759, - "cosine_similarity": 0.9994606971740723, - "kl_divergence": 0.0028384683547810416, - "sigma_level": 29.136462853420184, - "cpk": 9.455979125389131, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.4594208002090454, - "mean_abs_diff": 0.07261195033788681, - "cosine_similarity": 0.9997691512107849, - "kl_divergence": 0.0028756378761694195, - "sigma_level": 43.5227244869569, - "cpk": 14.244219003234265, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4376527667045593, - "mean_abs_diff": 0.04770943894982338, - "cosine_similarity": 0.9997802972793579, - "kl_divergence": 0.0002040249183579563, - "sigma_level": 65.76835287900975, - "cpk": 21.661303358293953, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3315706253051758, - "mean_abs_diff": 0.052924565970897675, - "cosine_similarity": 0.9998641014099121, - "kl_divergence": 0.0005738790430968991, - "sigma_level": 59.410746258878895, - "cpk": 19.54155808964644, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.4616411328315735, - "mean_abs_diff": 0.0820874571800232, - "cosine_similarity": 0.9993736147880554, - "kl_divergence": 0.000581956823405685, - "sigma_level": 39.33158269369345, - "cpk": 12.841475097048557, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.4140510559082031, - "mean_abs_diff": 0.08190098404884338, - "cosine_similarity": 0.9998055100440979, - "kl_divergence": 0.0045809146856644325, - "sigma_level": 39.888270533950774, - "cpk": 13.023849460588838, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.38747692108154297, - "mean_abs_diff": 0.06567694246768951, - "cosine_similarity": 0.9997859597206116, - "kl_divergence": 0.0007163080545155323, - "sigma_level": 48.645284717186506, - "cpk": 15.948855441920916, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.31238651275634766, - "mean_abs_diff": 0.046092480421066284, - "cosine_similarity": 0.9997801780700684, - "kl_divergence": 0.00007854547302267981, - "sigma_level": 67.973856818208, - "cpk": 22.396861967357868, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2587924003601074, - "mean_abs_diff": 0.04632105305790901, - "cosine_similarity": 0.9999061226844788, - "kl_divergence": 0.0006675378664987662, - "sigma_level": 67.8328996522191, - "cpk": 22.349125605417846, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.3423733711242676, - "mean_abs_diff": 0.0509999543428421, - "cosine_similarity": 0.9999075531959534, - "kl_divergence": 0.0013139015230752497, - "sigma_level": 61.44627485953518, - "cpk": 20.22094518548056, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 1075, - "max_abs_diff": 0.33937501907348633, - "mean_abs_diff": 0.05567406490445137, - "cosine_similarity": 0.9999013543128967, - "kl_divergence": 0.0023980062691288805, - "sigma_level": 57.02066163057764, - "cpk": 18.742339542149587, - "verdict": "WarnArgmax" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.30877065658569336, - "mean_abs_diff": 0.04503734037280083, - "cosine_similarity": 0.9998331665992737, - "kl_divergence": 0.00005757731281013704, - "sigma_level": 69.88200924330435, - "cpk": 23.031728094749294, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.34514331817626953, - "mean_abs_diff": 0.057505182921886444, - "cosine_similarity": 0.9999133348464966, - "kl_divergence": 0.0030512423213599734, - "sigma_level": 55.02250951493119, - "cpk": 18.07716321543728, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.39675191044807434, - "mean_abs_diff": 0.06584710627794266, - "cosine_similarity": 0.9995452761650085, - "kl_divergence": 0.002512817256532628, - "sigma_level": 47.60950042344511, - "cpk": 15.608587821629843, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.26343512535095215, - "mean_abs_diff": 0.04300215467810631, - "cosine_similarity": 0.9998367428779602, - "kl_divergence": 0.0007504716839459635, - "sigma_level": 73.06733630767575, - "cpk": 24.093941027740257, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.4165067672729492, - "mean_abs_diff": 0.0824187770485878, - "cosine_similarity": 0.9998924136161804, - "kl_divergence": 0.0009092099497840965, - "sigma_level": 40.712890472295875, - "cpk": 13.291337937195314, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.37496471405029297, - "mean_abs_diff": 0.07949145883321762, - "cosine_similarity": 0.9998467564582825, - "kl_divergence": 0.0021284045408820193, - "sigma_level": 42.300035778519224, - "cpk": 13.81980429677876, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.30150842666625977, - "mean_abs_diff": 0.05042034387588501, - "cosine_similarity": 0.9998810887336731, - "kl_divergence": 0.0015508871007383236, - "sigma_level": 61.98594278003023, - "cpk": 20.40153488080674, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.3162221908569336, - "mean_abs_diff": 0.050118133425712585, - "cosine_similarity": 0.9999067187309265, - "kl_divergence": 0.0020995741983847304, - "sigma_level": 63.78172992219015, - "cpk": 20.994191536533126, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.3547534942626953, - "mean_abs_diff": 0.05901937186717987, - "cosine_similarity": 0.9999095797538757, - "kl_divergence": 0.001992879373807011, - "sigma_level": 54.67646177583371, - "cpk": 17.956573056117104, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3257865905761719, - "mean_abs_diff": 0.04293311759829521, - "cosine_similarity": 0.9998363256454468, - "kl_divergence": 0.0003393573469818488, - "sigma_level": 72.82458829123628, - "cpk": 24.014313879315836, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3269984722137451, - "mean_abs_diff": 0.05209345370531082, - "cosine_similarity": 0.9998263120651245, - "kl_divergence": 0.0013256906000322828, - "sigma_level": 60.776719885309284, - "cpk": 19.995067524794262, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.3318147659301758, - "mean_abs_diff": 0.061590515077114105, - "cosine_similarity": 0.9999039769172668, - "kl_divergence": 0.0019106988855219238, - "sigma_level": 53.494702850667856, - "cpk": 17.557003758350135, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.2706027030944824, - "mean_abs_diff": 0.0453641451895237, - "cosine_similarity": 0.9998862147331238, - "kl_divergence": 0.0015863168518343554, - "sigma_level": 69.14568409477452, - "cpk": 22.787166793882815, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.320314884185791, - "mean_abs_diff": 0.05479338765144348, - "cosine_similarity": 0.9998805522918701, - "kl_divergence": 0.0018053862786531084, - "sigma_level": 58.55186723394565, - "cpk": 19.249934481393097, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.2487473487854004, - "mean_abs_diff": 0.04012826830148697, - "cosine_similarity": 0.9999148845672607, - "kl_divergence": 0.000992232432491818, - "sigma_level": 79.04261413932409, - "cpk": 26.083217777488557, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2852489948272705, - "mean_abs_diff": 0.04260895773768425, - "cosine_similarity": 0.9998908042907715, - "kl_divergence": 0.0007555272284797762, - "sigma_level": 74.2463000340896, - "cpk": 24.48513688966887, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.2984335422515869, - "mean_abs_diff": 0.044331666082143784, - "cosine_similarity": 0.9999169707298279, - "kl_divergence": 0.0029629589041888825, - "sigma_level": 70.46956643182976, - "cpk": 23.229519369942476, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 1030, - "max_abs_diff": 0.273104190826416, - "mean_abs_diff": 0.04725624620914459, - "cosine_similarity": 0.9999066591262817, - "kl_divergence": 0.0003615828044026549, - "sigma_level": 67.64386780596082, - "cpk": 22.281572996022163, - "verdict": "WarnArgmax" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.3086332678794861, - "mean_abs_diff": 0.04187058284878731, - "cosine_similarity": 0.9999241232872009, - "kl_divergence": 0.0016002433269929573, - "sigma_level": 74.5359265207267, - "cpk": 24.585236949692465, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2898261547088623, - "mean_abs_diff": 0.04877804219722748, - "cosine_similarity": 0.9998968243598938, - "kl_divergence": 0.0002295654035777167, - "sigma_level": 66.83978374060625, - "cpk": 22.008235097556014, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2525869607925415, - "mean_abs_diff": 0.038769688457250595, - "cosine_similarity": 0.9998642802238464, - "kl_divergence": 0.00015459832954484046, - "sigma_level": 81.89698771623543, - "cpk": 27.03440251379967, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.2620408535003662, - "mean_abs_diff": 0.038319140672683716, - "cosine_similarity": 0.9998922944068909, - "kl_divergence": 0.0003140919129558196, - "sigma_level": 82.67792301311985, - "cpk": 27.29529542416786, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.3415346145629883, - "mean_abs_diff": 0.07152623683214188, - "cosine_similarity": 0.9998765587806702, - "kl_divergence": 0.0012743173908936454, - "sigma_level": 47.2490865840439, - "cpk": 15.46806641492191, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.30021238327026367, - "mean_abs_diff": 0.050972770899534225, - "cosine_similarity": 0.9998793601989746, - "kl_divergence": 0.0007696301505989618, - "sigma_level": 63.01291429802815, - "cpk": 20.736642862323936, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.2957894802093506, - "mean_abs_diff": 0.042167410254478455, - "cosine_similarity": 0.9998956322669983, - "kl_divergence": 0.0000986269813444574, - "sigma_level": 74.45760637654595, - "cpk": 24.557561755961462, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.24124550819396973, - "mean_abs_diff": 0.040473658591508865, - "cosine_similarity": 0.9999110102653503, - "kl_divergence": 0.0005439255887544281, - "sigma_level": 77.81332168413442, - "cpk": 25.675324743401898, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.2903571128845215, - "mean_abs_diff": 0.05764927715063095, - "cosine_similarity": 0.9999284148216248, - "kl_divergence": 0.0006913008282435065, - "sigma_level": 56.68813858123516, - "cpk": 18.62371034272647, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2806577682495117, - "mean_abs_diff": 0.04559638351202011, - "cosine_similarity": 0.9998930096626282, - "kl_divergence": 0.00037460736880700403, - "sigma_level": 69.88090861817952, - "cpk": 23.028109813599595, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2667236328125, - "mean_abs_diff": 0.043514035642147064, - "cosine_similarity": 0.9999178051948547, - "kl_divergence": 0.0021676753013869826, - "sigma_level": 72.52037518933956, - "cpk": 23.910487213882288, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2771492004394531, - "mean_abs_diff": 0.0458214171230793, - "cosine_similarity": 0.9998854398727417, - "kl_divergence": 0.0005921675492852606, - "sigma_level": 69.37134539440211, - "cpk": 22.858890685325196, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2921719551086426, - "mean_abs_diff": 0.04640787094831467, - "cosine_similarity": 0.9999167919158936, - "kl_divergence": 0.0010064557477895723, - "sigma_level": 68.66264823251565, - "cpk": 22.62200880099321, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.2976968288421631, - "mean_abs_diff": 0.043953705579042435, - "cosine_similarity": 0.9999005198478699, - "kl_divergence": 0.002171449635300438, - "sigma_level": 71.44745250213712, - "cpk": 23.554119143074743, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23976421356201172, - "mean_abs_diff": 0.041163649410009384, - "cosine_similarity": 0.9998940229415894, - "kl_divergence": 0.00028813644654819, - "sigma_level": 76.62883812597585, - "cpk": 25.280085823049113, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.28889644145965576, - "mean_abs_diff": 0.05656943470239639, - "cosine_similarity": 0.9998864531517029, - "kl_divergence": 0.0014977009727720507, - "sigma_level": 57.23183046776684, - "cpk": 18.8074791312102, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.29786229133605957, - "mean_abs_diff": 0.049185607582330704, - "cosine_similarity": 0.9998613595962524, - "kl_divergence": 0.00020494337876287373, - "sigma_level": 64.53610010140734, - "cpk": 21.247512759262296, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.3566131591796875, - "mean_abs_diff": 0.04558814316987991, - "cosine_similarity": 0.999882161617279, - "kl_divergence": 0.000034609400973773357, - "sigma_level": 68.88439368954486, - "cpk": 22.69977192970751, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.27669310569763184, - "mean_abs_diff": 0.041648074984550476, - "cosine_similarity": 0.9999054074287415, - "kl_divergence": 0.00014012989035636092, - "sigma_level": 75.50820301344545, - "cpk": 24.907336729394096, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.28789544105529785, - "mean_abs_diff": 0.042276348918676376, - "cosine_similarity": 0.9999191164970398, - "kl_divergence": 0.00047324870857447856, - "sigma_level": 74.10473544425255, - "cpk": 24.440505343736902, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.28496575355529785, - "mean_abs_diff": 0.05814102292060852, - "cosine_similarity": 0.999931812286377, - "kl_divergence": 0.0006591423793313406, - "sigma_level": 56.58944499601091, - "cpk": 18.58896764712217, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.27792835235595703, - "mean_abs_diff": 0.04199598729610443, - "cosine_similarity": 0.9999072551727295, - "kl_divergence": 0.001638797341847131, - "sigma_level": 74.50897424492256, - "cpk": 24.57556825365456, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.3433370590209961, - "mean_abs_diff": 0.053227003663778305, - "cosine_similarity": 0.9998949766159058, - "kl_divergence": 0.000473337168661806, - "sigma_level": 59.86797465225015, - "cpk": 19.69044214190352, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 1865, - "gpu_argmax": 6509, - "max_abs_diff": 0.3208746016025543, - "mean_abs_diff": 0.04410288855433464, - "cosine_similarity": 0.9999158382415771, - "kl_divergence": 0.0011789868514640338, - "sigma_level": 70.86618468274246, - "cpk": 23.361611273802996, - "verdict": "WarnArgmax" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.24564409255981445, - "mean_abs_diff": 0.04126206785440445, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.00040468322607162304, - "sigma_level": 75.96218013223144, - "cpk": 25.05953032482843, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.2946641445159912, - "mean_abs_diff": 0.04794362932443619, - "cosine_similarity": 0.9998166561126709, - "kl_divergence": 0.0009396785610877979, - "sigma_level": 65.26452919527235, - "cpk": 21.494091532109785, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 15, + "gpu_argmax": 16, + "max_abs_diff": 12.008713722229004, + "mean_abs_diff": 1.4603080749511719, + "cosine_similarity": 0.9506108164787292, + "kl_divergence": 5.453976683901305, + "sigma_level": 2.138399453406798, + "cpk": 0.4525729853621727, + "verdict": "WarnOutOfSpec" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.4038071632385254, + "mean_abs_diff": 0.06531447172164917, + "cosine_similarity": 0.9998307228088379, + "kl_divergence": 3.169573582524447e-05, + "sigma_level": 48.145552545660266, + "cpk": 15.786467404364531, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.47008025646209717, + "mean_abs_diff": 0.079141765832901, + "cosine_similarity": 0.9998854398727417, + "kl_divergence": 2.350304035683132e-06, + "sigma_level": 40.26894361164864, + "cpk": 13.157401595078596, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.34066465497016907, + "mean_abs_diff": 0.05701277032494545, + "cosine_similarity": 0.9998948574066162, + "kl_divergence": 7.930019129213315e-05, + "sigma_level": 55.01750807501765, + "cpk": 18.0777776456949, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.36131715774536133, + "mean_abs_diff": 0.06654312461614609, + "cosine_similarity": 0.9998743534088135, + "kl_divergence": 5.586878639155519e-06, + "sigma_level": 48.151386331009974, + "cpk": 15.783450135247941, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.39881277084350586, + "mean_abs_diff": 0.05531281977891922, + "cosine_similarity": 0.9998670220375061, + "kl_divergence": 0.00012843890921633306, + "sigma_level": 56.67790489208867, + "cpk": 18.631383735800988, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.40886592864990234, + "mean_abs_diff": 0.06747263669967651, + "cosine_similarity": 0.9999094009399414, + "kl_divergence": 1.1448242344448684e-05, + "sigma_level": 47.16232087734927, + "cpk": 15.455593113910512, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.45634615421295166, + "mean_abs_diff": 0.06790852546691895, + "cosine_similarity": 0.9999128580093384, + "kl_divergence": 4.6069409166491056e-06, + "sigma_level": 46.829472258467334, + "cpk": 15.34481405203357, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3148663640022278, + "mean_abs_diff": 0.05503924563527107, + "cosine_similarity": 0.9998142719268799, + "kl_divergence": 0.0006964580484037847, + "sigma_level": 58.33280314138458, + "cpk": 19.17671825707048, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.37107133865356445, + "mean_abs_diff": 0.049693331122398376, + "cosine_similarity": 0.9998795390129089, + "kl_divergence": 0.0019487691332511425, + "sigma_level": 63.28510665986591, + "cpk": 20.832964906591553, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.3408195972442627, + "mean_abs_diff": 0.049113236367702484, + "cosine_similarity": 0.9999272227287292, + "kl_divergence": 0.0023197360102891846, + "sigma_level": 63.60800653625746, + "cpk": 20.942335923761355, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.2871994972229004, + "mean_abs_diff": 0.04618430510163307, + "cosine_similarity": 0.99989253282547, + "kl_divergence": 0.0010725620753604848, + "sigma_level": 67.48480073589947, + "cpk": 22.235205359724013, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.30663490295410156, + "mean_abs_diff": 0.05332659184932709, + "cosine_similarity": 0.9998827576637268, + "kl_divergence": 0.0003849714682721958, + "sigma_level": 60.14815823956778, + "cpk": 19.782094722778414, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.341217041015625, + "mean_abs_diff": 0.05129178240895271, + "cosine_similarity": 0.9998595714569092, + "kl_divergence": 0.0011121437754711468, + "sigma_level": 60.927599133008556, + "cpk": 20.048775947883673, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.3473668098449707, + "mean_abs_diff": 0.06534294039011002, + "cosine_similarity": 0.999890148639679, + "kl_divergence": 0.0018708287004877242, + "sigma_level": 50.02631084841449, + "cpk": 16.403031428829408, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3845231533050537, + "mean_abs_diff": 0.05668144300580025, + "cosine_similarity": 0.9998037815093994, + "kl_divergence": 0.0014187455405557119, + "sigma_level": 56.08514037244682, + "cpk": 18.43013123352451, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3365215063095093, + "mean_abs_diff": 0.051528360694646835, + "cosine_similarity": 0.999897837638855, + "kl_divergence": 0.001462163794542408, + "sigma_level": 61.80499140623471, + "cpk": 20.336271311252403, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.34671688079833984, + "mean_abs_diff": 0.05129155516624451, + "cosine_similarity": 0.999882161617279, + "kl_divergence": 8.450073376790388e-05, + "sigma_level": 61.06263290998148, + "cpk": 20.09321118628562, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.3720208406448364, + "mean_abs_diff": 0.05188516154885292, + "cosine_similarity": 0.9998630881309509, + "kl_divergence": 0.0006878276376716025, + "sigma_level": 60.17461919193516, + "cpk": 19.79802557748553, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.364626407623291, + "mean_abs_diff": 0.077647864818573, + "cosine_similarity": 0.9997988343238831, + "kl_divergence": 0.004185557706680174, + "sigma_level": 43.092094054533995, + "cpk": 14.085197260353358, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.4366130828857422, + "mean_abs_diff": 0.0633498802781105, + "cosine_similarity": 0.9997689127922058, + "kl_divergence": 0.0032265442609676616, + "sigma_level": 49.67682516705651, + "cpk": 16.296689978441368, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 1.0816888809204102, + "mean_abs_diff": 0.16485518217086792, + "cosine_similarity": 0.9990662932395935, + "kl_divergence": 0.013121373568509257, + "sigma_level": 19.17278201676311, + "cpk": 6.1275329662463855, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.8328787088394165, + "mean_abs_diff": 0.12702322006225586, + "cosine_similarity": 0.9992517232894897, + "kl_divergence": 0.021413090149460762, + "sigma_level": 24.71860116353615, + "cpk": 7.977880694909801, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.8862452507019043, + "mean_abs_diff": 0.10550703853368759, + "cosine_similarity": 0.9994606971740723, + "kl_divergence": 0.0028384683547810416, + "sigma_level": 29.136462853420184, + "cpk": 9.455979125389131, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.4594208002090454, + "mean_abs_diff": 0.07261195033788681, + "cosine_similarity": 0.9997691512107849, + "kl_divergence": 0.0028756378761694195, + "sigma_level": 43.5227244869569, + "cpk": 14.244219003234265, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4376527667045593, + "mean_abs_diff": 0.04770943894982338, + "cosine_similarity": 0.9997802972793579, + "kl_divergence": 0.0002040249183579563, + "sigma_level": 65.76835287900975, + "cpk": 21.661303358293953, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3315706253051758, + "mean_abs_diff": 0.052924565970897675, + "cosine_similarity": 0.9998641014099121, + "kl_divergence": 0.0005738790430968991, + "sigma_level": 59.410746258878895, + "cpk": 19.54155808964644, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.4616411328315735, + "mean_abs_diff": 0.0820874571800232, + "cosine_similarity": 0.9993736147880554, + "kl_divergence": 0.000581956823405685, + "sigma_level": 39.33158269369345, + "cpk": 12.841475097048557, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.4140510559082031, + "mean_abs_diff": 0.08190098404884338, + "cosine_similarity": 0.9998055100440979, + "kl_divergence": 0.0045809146856644325, + "sigma_level": 39.888270533950774, + "cpk": 13.023849460588838, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.38747692108154297, + "mean_abs_diff": 0.06567694246768951, + "cosine_similarity": 0.9997859597206116, + "kl_divergence": 0.0007163080545155323, + "sigma_level": 48.645284717186506, + "cpk": 15.948855441920916, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.31238651275634766, + "mean_abs_diff": 0.046092480421066284, + "cosine_similarity": 0.9997801780700684, + "kl_divergence": 7.854547302267981e-05, + "sigma_level": 67.973856818208, + "cpk": 22.396861967357868, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2587924003601074, + "mean_abs_diff": 0.04632105305790901, + "cosine_similarity": 0.9999061226844788, + "kl_divergence": 0.0006675378664987662, + "sigma_level": 67.8328996522191, + "cpk": 22.349125605417846, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.3423733711242676, + "mean_abs_diff": 0.0509999543428421, + "cosine_similarity": 0.9999075531959534, + "kl_divergence": 0.0013139015230752497, + "sigma_level": 61.44627485953518, + "cpk": 20.22094518548056, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 1075, + "max_abs_diff": 0.33937501907348633, + "mean_abs_diff": 0.05567406490445137, + "cosine_similarity": 0.9999013543128967, + "kl_divergence": 0.0023980062691288805, + "sigma_level": 57.02066163057764, + "cpk": 18.742339542149587, + "verdict": "WarnArgmax" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.30877065658569336, + "mean_abs_diff": 0.04503734037280083, + "cosine_similarity": 0.9998331665992737, + "kl_divergence": 5.757731281013704e-05, + "sigma_level": 69.88200924330435, + "cpk": 23.031728094749294, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.34514331817626953, + "mean_abs_diff": 0.057505182921886444, + "cosine_similarity": 0.9999133348464966, + "kl_divergence": 0.0030512423213599734, + "sigma_level": 55.02250951493119, + "cpk": 18.07716321543728, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.39675191044807434, + "mean_abs_diff": 0.06584710627794266, + "cosine_similarity": 0.9995452761650085, + "kl_divergence": 0.002512817256532628, + "sigma_level": 47.60950042344511, + "cpk": 15.608587821629843, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.26343512535095215, + "mean_abs_diff": 0.04300215467810631, + "cosine_similarity": 0.9998367428779602, + "kl_divergence": 0.0007504716839459635, + "sigma_level": 73.06733630767575, + "cpk": 24.093941027740257, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.4165067672729492, + "mean_abs_diff": 0.0824187770485878, + "cosine_similarity": 0.9998924136161804, + "kl_divergence": 0.0009092099497840965, + "sigma_level": 40.712890472295875, + "cpk": 13.291337937195314, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.37496471405029297, + "mean_abs_diff": 0.07949145883321762, + "cosine_similarity": 0.9998467564582825, + "kl_divergence": 0.0021284045408820193, + "sigma_level": 42.300035778519224, + "cpk": 13.81980429677876, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.30150842666625977, + "mean_abs_diff": 0.05042034387588501, + "cosine_similarity": 0.9998810887336731, + "kl_divergence": 0.0015508871007383236, + "sigma_level": 61.98594278003023, + "cpk": 20.40153488080674, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.3162221908569336, + "mean_abs_diff": 0.050118133425712585, + "cosine_similarity": 0.9999067187309265, + "kl_divergence": 0.0020995741983847304, + "sigma_level": 63.78172992219015, + "cpk": 20.994191536533126, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.3547534942626953, + "mean_abs_diff": 0.05901937186717987, + "cosine_similarity": 0.9999095797538757, + "kl_divergence": 0.001992879373807011, + "sigma_level": 54.67646177583371, + "cpk": 17.956573056117104, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3257865905761719, + "mean_abs_diff": 0.04293311759829521, + "cosine_similarity": 0.9998363256454468, + "kl_divergence": 0.0003393573469818488, + "sigma_level": 72.82458829123628, + "cpk": 24.014313879315836, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3269984722137451, + "mean_abs_diff": 0.05209345370531082, + "cosine_similarity": 0.9998263120651245, + "kl_divergence": 0.0013256906000322828, + "sigma_level": 60.776719885309284, + "cpk": 19.995067524794262, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.3318147659301758, + "mean_abs_diff": 0.061590515077114105, + "cosine_similarity": 0.9999039769172668, + "kl_divergence": 0.0019106988855219238, + "sigma_level": 53.494702850667856, + "cpk": 17.557003758350135, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.2706027030944824, + "mean_abs_diff": 0.0453641451895237, + "cosine_similarity": 0.9998862147331238, + "kl_divergence": 0.0015863168518343554, + "sigma_level": 69.14568409477452, + "cpk": 22.787166793882815, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.320314884185791, + "mean_abs_diff": 0.05479338765144348, + "cosine_similarity": 0.9998805522918701, + "kl_divergence": 0.0018053862786531084, + "sigma_level": 58.55186723394565, + "cpk": 19.249934481393097, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.2487473487854004, + "mean_abs_diff": 0.04012826830148697, + "cosine_similarity": 0.9999148845672607, + "kl_divergence": 0.000992232432491818, + "sigma_level": 79.04261413932409, + "cpk": 26.083217777488557, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2852489948272705, + "mean_abs_diff": 0.04260895773768425, + "cosine_similarity": 0.9998908042907715, + "kl_divergence": 0.0007555272284797762, + "sigma_level": 74.2463000340896, + "cpk": 24.48513688966887, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.2984335422515869, + "mean_abs_diff": 0.044331666082143784, + "cosine_similarity": 0.9999169707298279, + "kl_divergence": 0.0029629589041888825, + "sigma_level": 70.46956643182976, + "cpk": 23.229519369942476, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 1030, + "max_abs_diff": 0.273104190826416, + "mean_abs_diff": 0.04725624620914459, + "cosine_similarity": 0.9999066591262817, + "kl_divergence": 0.0003615828044026549, + "sigma_level": 67.64386780596082, + "cpk": 22.281572996022163, + "verdict": "WarnArgmax" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.3086332678794861, + "mean_abs_diff": 0.04187058284878731, + "cosine_similarity": 0.9999241232872009, + "kl_divergence": 0.0016002433269929573, + "sigma_level": 74.5359265207267, + "cpk": 24.585236949692465, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2898261547088623, + "mean_abs_diff": 0.04877804219722748, + "cosine_similarity": 0.9998968243598938, + "kl_divergence": 0.0002295654035777167, + "sigma_level": 66.83978374060625, + "cpk": 22.008235097556014, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2525869607925415, + "mean_abs_diff": 0.038769688457250595, + "cosine_similarity": 0.9998642802238464, + "kl_divergence": 0.00015459832954484046, + "sigma_level": 81.89698771623543, + "cpk": 27.03440251379967, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.2620408535003662, + "mean_abs_diff": 0.038319140672683716, + "cosine_similarity": 0.9998922944068909, + "kl_divergence": 0.0003140919129558196, + "sigma_level": 82.67792301311985, + "cpk": 27.29529542416786, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.3415346145629883, + "mean_abs_diff": 0.07152623683214188, + "cosine_similarity": 0.9998765587806702, + "kl_divergence": 0.0012743173908936454, + "sigma_level": 47.2490865840439, + "cpk": 15.46806641492191, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.30021238327026367, + "mean_abs_diff": 0.050972770899534225, + "cosine_similarity": 0.9998793601989746, + "kl_divergence": 0.0007696301505989618, + "sigma_level": 63.01291429802815, + "cpk": 20.736642862323936, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.2957894802093506, + "mean_abs_diff": 0.042167410254478455, + "cosine_similarity": 0.9998956322669983, + "kl_divergence": 9.86269813444574e-05, + "sigma_level": 74.45760637654595, + "cpk": 24.557561755961462, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.24124550819396973, + "mean_abs_diff": 0.040473658591508865, + "cosine_similarity": 0.9999110102653503, + "kl_divergence": 0.0005439255887544281, + "sigma_level": 77.81332168413442, + "cpk": 25.675324743401898, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.2903571128845215, + "mean_abs_diff": 0.05764927715063095, + "cosine_similarity": 0.9999284148216248, + "kl_divergence": 0.0006913008282435065, + "sigma_level": 56.68813858123516, + "cpk": 18.62371034272647, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2806577682495117, + "mean_abs_diff": 0.04559638351202011, + "cosine_similarity": 0.9998930096626282, + "kl_divergence": 0.00037460736880700403, + "sigma_level": 69.88090861817952, + "cpk": 23.028109813599595, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2667236328125, + "mean_abs_diff": 0.043514035642147064, + "cosine_similarity": 0.9999178051948547, + "kl_divergence": 0.0021676753013869826, + "sigma_level": 72.52037518933956, + "cpk": 23.910487213882288, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2771492004394531, + "mean_abs_diff": 0.0458214171230793, + "cosine_similarity": 0.9998854398727417, + "kl_divergence": 0.0005921675492852606, + "sigma_level": 69.37134539440211, + "cpk": 22.858890685325196, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2921719551086426, + "mean_abs_diff": 0.04640787094831467, + "cosine_similarity": 0.9999167919158936, + "kl_divergence": 0.0010064557477895723, + "sigma_level": 68.66264823251565, + "cpk": 22.62200880099321, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.2976968288421631, + "mean_abs_diff": 0.043953705579042435, + "cosine_similarity": 0.9999005198478699, + "kl_divergence": 0.002171449635300438, + "sigma_level": 71.44745250213712, + "cpk": 23.554119143074743, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23976421356201172, + "mean_abs_diff": 0.041163649410009384, + "cosine_similarity": 0.9998940229415894, + "kl_divergence": 0.00028813644654819, + "sigma_level": 76.62883812597585, + "cpk": 25.280085823049113, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.28889644145965576, + "mean_abs_diff": 0.05656943470239639, + "cosine_similarity": 0.9998864531517029, + "kl_divergence": 0.0014977009727720507, + "sigma_level": 57.23183046776684, + "cpk": 18.8074791312102, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.29786229133605957, + "mean_abs_diff": 0.049185607582330704, + "cosine_similarity": 0.9998613595962524, + "kl_divergence": 0.00020494337876287373, + "sigma_level": 64.53610010140734, + "cpk": 21.247512759262296, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.3566131591796875, + "mean_abs_diff": 0.04558814316987991, + "cosine_similarity": 0.999882161617279, + "kl_divergence": 3.4609400973773357e-05, + "sigma_level": 68.88439368954486, + "cpk": 22.69977192970751, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.27669310569763184, + "mean_abs_diff": 0.041648074984550476, + "cosine_similarity": 0.9999054074287415, + "kl_divergence": 0.00014012989035636092, + "sigma_level": 75.50820301344545, + "cpk": 24.907336729394096, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.28789544105529785, + "mean_abs_diff": 0.042276348918676376, + "cosine_similarity": 0.9999191164970398, + "kl_divergence": 0.00047324870857447856, + "sigma_level": 74.10473544425255, + "cpk": 24.440505343736902, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.28496575355529785, + "mean_abs_diff": 0.05814102292060852, + "cosine_similarity": 0.999931812286377, + "kl_divergence": 0.0006591423793313406, + "sigma_level": 56.58944499601091, + "cpk": 18.58896764712217, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.27792835235595703, + "mean_abs_diff": 0.04199598729610443, + "cosine_similarity": 0.9999072551727295, + "kl_divergence": 0.001638797341847131, + "sigma_level": 74.50897424492256, + "cpk": 24.57556825365456, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.3433370590209961, + "mean_abs_diff": 0.053227003663778305, + "cosine_similarity": 0.9998949766159058, + "kl_divergence": 0.000473337168661806, + "sigma_level": 59.86797465225015, + "cpk": 19.69044214190352, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 1865, + "gpu_argmax": 6509, + "max_abs_diff": 0.3208746016025543, + "mean_abs_diff": 0.04410288855433464, + "cosine_similarity": 0.9999158382415771, + "kl_divergence": 0.0011789868514640338, + "sigma_level": 70.86618468274246, + "cpk": 23.361611273802996, + "verdict": "WarnArgmax" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.24564409255981445, + "mean_abs_diff": 0.04126206785440445, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.00040468322607162304, + "sigma_level": 75.96218013223144, + "cpk": 25.05953032482843, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.2946641445159912, + "mean_abs_diff": 0.04794362932443619, + "cosine_similarity": 0.9998166561126709, + "kl_divergence": 0.0009396785610877979, + "sigma_level": 65.26452919527235, + "cpk": 21.494091532109785, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json index a855176590..094462a85e 100644 --- a/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/gx10/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "21d182d69505159c", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1/gx10/RECORD.md heading '2026-09-08T12:5xZ' \u2014 hour stated, minute not", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9984647035598755, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.8031983375549316, - "mean_abs_diff": 0.12188125401735306, - "cosine_similarity": 0.9984647035598755, - "kl_divergence": 0.011949143213562554, - "sigma_level": 26.330537211297408, - "cpk": 8.50941249591051, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.24585700035095215, - "mean_abs_diff": 0.03762512281537056, - "cosine_similarity": 0.9998586177825928, - "kl_divergence": 0.000013277331549664009, - "sigma_level": 84.0184221427533, - "cpk": 27.742707093261536, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.2099609375, - "mean_abs_diff": 0.033425211906433105, - "cosine_similarity": 0.9999147653579712, - "kl_divergence": 1.5090158264594303e-6, - "sigma_level": 94.57560609882762, - "cpk": 31.261767893356485, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.22704386711120605, - "mean_abs_diff": 0.0315629281103611, - "cosine_similarity": 0.999933660030365, - "kl_divergence": 0.000037310540638633336, - "sigma_level": 99.18469092152263, - "cpk": 32.80068370140717, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.2088146209716797, - "mean_abs_diff": 0.03077378123998642, - "cosine_similarity": 0.9999191164970398, - "kl_divergence": 1.4999781410245668e-6, - "sigma_level": 101.59603971600392, - "cpk": 33.60480538024553, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2547646760940552, - "mean_abs_diff": 0.03083600476384163, - "cosine_similarity": 0.9999075531959534, - "kl_divergence": 0.0002142816112921191, - "sigma_level": 101.00436248100775, - "cpk": 33.40857324344982, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.20438094437122345, - "mean_abs_diff": 0.031773317605257034, - "cosine_similarity": 0.9999481439590454, - "kl_divergence": 9.281385733172737e-7, - "sigma_level": 99.44615585236863, - "cpk": 32.88540742624628, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.18281269073486328, - "mean_abs_diff": 0.030860792845487595, - "cosine_similarity": 0.9999558925628662, - "kl_divergence": 2.0817027272510226e-6, - "sigma_level": 102.06207588666598, - "cpk": 33.75821558044542, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.28357791900634766, - "mean_abs_diff": 0.03390377387404442, - "cosine_similarity": 0.999890148639679, - "kl_divergence": 0.00045742262385036316, - "sigma_level": 92.77850864421463, - "cpk": 30.6640410832845, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2932462692260742, - "mean_abs_diff": 0.044488128274679184, - "cosine_similarity": 0.9998958706855774, - "kl_divergence": 0.0016326477262920424, - "sigma_level": 71.4951399913799, - "cpk": 23.566656250547247, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.25107574462890625, - "mean_abs_diff": 0.036512766033411026, - "cosine_similarity": 0.9999284148216248, - "kl_divergence": 0.0011823933043749269, - "sigma_level": 85.6818989695192, - "cpk": 28.299926062308696, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.25175952911376953, - "mean_abs_diff": 0.04251187667250633, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.000520927378713784, - "sigma_level": 75.90532316155377, - "cpk": 25.032867909098698, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.23739862442016602, - "mean_abs_diff": 0.035071682184934616, - "cosine_similarity": 0.9999188184738159, - "kl_divergence": 0.00016192173524096987, - "sigma_level": 91.2163777694204, - "cpk": 30.13886660554096, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.20715034008026123, - "mean_abs_diff": 0.031439900398254395, - "cosine_similarity": 0.9999179244041443, - "kl_divergence": 0.0007082983479816206, - "sigma_level": 100.4901828181864, - "cpk": 33.23344416116162, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.23068474233150482, - "mean_abs_diff": 0.03183136135339737, - "cosine_similarity": 0.9999409914016724, - "kl_divergence": 0.0007547449349859649, - "sigma_level": 98.39571148132515, - "cpk": 32.53756470645949, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23305463790893555, - "mean_abs_diff": 0.028933987021446228, - "cosine_similarity": 0.9998893141746521, - "kl_divergence": 0.0003845604489353084, - "sigma_level": 107.47455288493924, - "cpk": 35.565712018454036, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2608985900878906, - "mean_abs_diff": 0.04424131289124489, - "cosine_similarity": 0.9999105930328369, - "kl_divergence": 0.0007788479711618704, - "sigma_level": 73.24530032745929, - "cpk": 24.14506108835307, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.2301793098449707, - "mean_abs_diff": 0.0324413925409317, - "cosine_similarity": 0.9999143481254578, - "kl_divergence": 0.00002250089535157956, - "sigma_level": 96.25744820928855, - "cpk": 31.825588931234023, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.19923532009124756, - "mean_abs_diff": 0.03906163200736046, - "cosine_similarity": 0.999877393245697, - "kl_divergence": 0.0005163370630241004, - "sigma_level": 83.7633797367138, - "cpk": 27.648465385990573, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 5942, - "gpu_argmax": 5942, - "max_abs_diff": 0.17966842651367188, - "mean_abs_diff": 0.02848205156624317, - "cosine_similarity": 0.9999198317527771, - "kl_divergence": 0.0008812988738490747, - "sigma_level": 109.6664749254824, - "cpk": 36.29519779233449, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.29142284393310547, - "mean_abs_diff": 0.043598152697086334, - "cosine_similarity": 0.9998776316642761, - "kl_divergence": 0.0010248388343692954, - "sigma_level": 73.4393195777277, - "cpk": 24.21295497016591, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.34327876567840576, - "mean_abs_diff": 0.0488663986325264, - "cosine_similarity": 0.9999039173126221, - "kl_divergence": 0.0018092925711578334, - "sigma_level": 64.72170503736106, - "cpk": 21.310341959242646, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3739614486694336, - "mean_abs_diff": 0.04563745856285095, - "cosine_similarity": 0.9998410940170288, - "kl_divergence": 0.0013028657116986625, - "sigma_level": 68.52047136061793, - "cpk": 22.579565439170377, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.25923848152160645, - "mean_abs_diff": 0.04782666265964508, - "cosine_similarity": 0.9998435378074646, - "kl_divergence": 0.00017440859928140556, - "sigma_level": 68.0917989073902, - "cpk": 22.425882677777388, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.20351624488830566, - "mean_abs_diff": 0.030075596645474434, - "cosine_similarity": 0.9998764991760254, - "kl_divergence": 0.0005101532884886966, - "sigma_level": 103.98703355346954, - "cpk": 34.40172184469705, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.20961999893188477, - "mean_abs_diff": 0.0313001312315464, - "cosine_similarity": 0.9998053312301636, - "kl_divergence": 0.000028302109163839864, - "sigma_level": 102.59202935793, - "cpk": 33.929747787458844, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19363737106323242, - "mean_abs_diff": 0.033073790371418, - "cosine_similarity": 0.9998298287391663, - "kl_divergence": 0.0005171189703267121, - "sigma_level": 96.9427542695765, - "cpk": 32.04706272879717, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.20902681350708008, - "mean_abs_diff": 0.03569749742746353, - "cosine_similarity": 0.9997431039810181, - "kl_divergence": 0.0001146219937046406, - "sigma_level": 90.59358669300643, - "cpk": 29.92836520367562, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.20511221885681152, - "mean_abs_diff": 0.03204226866364479, - "cosine_similarity": 0.9998698830604553, - "kl_divergence": 0.0009168103274381441, - "sigma_level": 97.91782700745156, - "cpk": 32.3778165591561, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.20564067363739014, - "mean_abs_diff": 0.03282441571354866, - "cosine_similarity": 0.9998794794082642, - "kl_divergence": 0.00003538160864634364, - "sigma_level": 96.38664467254054, - "cpk": 31.865228616349718, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2244204878807068, - "mean_abs_diff": 0.048710912466049194, - "cosine_similarity": 0.9996637105941772, - "kl_divergence": 3.898172129763024e-6, - "sigma_level": 69.19204623397793, - "cpk": 22.7831481023718, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19256067276000977, - "mean_abs_diff": 0.038800157606601715, - "cosine_similarity": 0.9998534321784973, - "kl_divergence": 0.00014612402448690114, - "sigma_level": 83.81763111733832, - "cpk": 27.668198930982378, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.24395322799682617, - "mean_abs_diff": 0.035089071840047836, - "cosine_similarity": 0.9998824596405029, - "kl_divergence": 0.0007183325325718559, - "sigma_level": 90.07186375980564, - "cpk": 29.76057641174898, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.22237825393676758, - "mean_abs_diff": 0.030163198709487915, - "cosine_similarity": 0.9998973608016968, - "kl_divergence": 0.0007083169726316857, - "sigma_level": 103.65308835461354, - "cpk": 34.290487059796824, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.19781363010406494, - "mean_abs_diff": 0.028445789590477943, - "cosine_similarity": 0.9998837113380432, - "kl_divergence": 9.986173050612662e-6, - "sigma_level": 109.84427252512704, - "cpk": 36.35437358637827, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.21171927452087402, - "mean_abs_diff": 0.0331043042242527, - "cosine_similarity": 0.999922513961792, - "kl_divergence": 0.0005099449753999734, - "sigma_level": 95.18447747645524, - "cpk": 31.465574500501155, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.2000875473022461, - "mean_abs_diff": 0.028575299307703972, - "cosine_similarity": 0.9998475313186646, - "kl_divergence": 0.001018203258176856, - "sigma_level": 109.12304284617422, - "cpk": 36.11449564783334, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.18700814247131348, - "mean_abs_diff": 0.026265908032655716, - "cosine_similarity": 0.9998998045921326, - "kl_divergence": 0.00046225863274036456, - "sigma_level": 119.02215463941639, - "cpk": 39.413532799171506, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.22919845581054688, - "mean_abs_diff": 0.04531732201576233, - "cosine_similarity": 0.9998840689659119, - "kl_divergence": 0.0008305399123272189, - "sigma_level": 72.06274536754663, - "cpk": 23.74877423608546, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23381578922271729, - "mean_abs_diff": 0.030515603721141815, - "cosine_similarity": 0.9999016523361206, - "kl_divergence": 0.0005923224650756628, - "sigma_level": 103.29603430637736, - "cpk": 34.169333031387545, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.19087600708007812, - "mean_abs_diff": 0.030218342319130898, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.00045970970727517754, - "sigma_level": 104.27458896076459, - "cpk": 34.49561255155461, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2747793197631836, - "mean_abs_diff": 0.03750928118824959, - "cosine_similarity": 0.9999193549156189, - "kl_divergence": 0.00287010815315867, - "sigma_level": 84.41037393510442, - "cpk": 27.872943607440046, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.18088853359222412, - "mean_abs_diff": 0.029429737478494644, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.0002790777073941985, - "sigma_level": 106.92372344549646, - "cpk": 35.37901305589681, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.200392484664917, - "mean_abs_diff": 0.03397694602608681, - "cosine_similarity": 0.9998542070388794, - "kl_divergence": 0.0003240177982787965, - "sigma_level": 94.48294788690083, - "cpk": 31.226795793905374, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.29866647720336914, - "mean_abs_diff": 0.045978154987096786, - "cosine_similarity": 0.9998794198036194, - "kl_divergence": 0.0009558542296503378, - "sigma_level": 68.50529301922718, - "cpk": 22.572618758086186, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.21673345565795898, - "mean_abs_diff": 0.03529226407408714, - "cosine_similarity": 0.9999400973320007, - "kl_divergence": 0.002097586731727194, - "sigma_level": 90.45730879559231, - "cpk": 29.886399329410327, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.22467482089996338, - "mean_abs_diff": 0.03361072018742561, - "cosine_similarity": 0.999942421913147, - "kl_divergence": 0.0008436583035788792, - "sigma_level": 93.8695872897644, - "cpk": 31.026943727212682, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.22781848907470703, - "mean_abs_diff": 0.030904922634363174, - "cosine_similarity": 0.9999483227729797, - "kl_divergence": 0.0008333849906099469, - "sigma_level": 102.17588221636045, - "cpk": 33.795482594203946, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1783003807067871, - "mean_abs_diff": 0.029902499169111252, - "cosine_similarity": 0.9999409317970276, - "kl_divergence": 0.0003094355721386634, - "sigma_level": 106.19563387482063, - "cpk": 35.13391838713145, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2627032399177551, - "mean_abs_diff": 0.03310393542051315, - "cosine_similarity": 0.9999225735664368, - "kl_divergence": 0.00015295545004923058, - "sigma_level": 94.89749613646042, - "cpk": 31.37070866351432, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.19065427780151367, - "mean_abs_diff": 0.028850441798567772, - "cosine_similarity": 0.9999325275421143, - "kl_divergence": 0.00038527130067871274, - "sigma_level": 108.36687717839267, - "cpk": 35.861756369220245, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.23432064056396484, - "mean_abs_diff": 0.037413571029901505, - "cosine_similarity": 0.9999155402183533, - "kl_divergence": 0.0002889698267149911, - "sigma_level": 85.30887203134782, - "cpk": 28.17031488184714, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.18627315759658813, - "mean_abs_diff": 0.02875906601548195, - "cosine_similarity": 0.9999480843544006, - "kl_divergence": 0.0007701578612177225, - "sigma_level": 108.96690810906699, - "cpk": 36.06115382770469, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.22917795181274414, - "mean_abs_diff": 0.03641311824321747, - "cosine_similarity": 0.9999172687530518, - "kl_divergence": 0.0005103069102512929, - "sigma_level": 87.72677177564587, - "cpk": 28.97605681573509, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.24593007564544678, - "mean_abs_diff": 0.03212062641978264, - "cosine_similarity": 0.9998915791511536, - "kl_divergence": 0.00003151487057192769, - "sigma_level": 97.4480464319994, - "cpk": 32.221841119434806, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.19705867767333984, - "mean_abs_diff": 0.0312761552631855, - "cosine_similarity": 0.9999309778213501, - "kl_divergence": 0.0001150733661477417, - "sigma_level": 100.417536631287, - "cpk": 33.210789338193436, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.1876506805419922, - "mean_abs_diff": 0.031015293672680855, - "cosine_similarity": 0.9999198913574219, - "kl_divergence": 0.0007250441446097092, - "sigma_level": 101.70368336368918, - "cpk": 33.638363653969876, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.19601619243621826, - "mean_abs_diff": 0.03426588699221611, - "cosine_similarity": 0.999900221824646, - "kl_divergence": 0.00023789191270330738, - "sigma_level": 93.12063513483277, - "cpk": 30.77430661492979, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.20610380172729492, - "mean_abs_diff": 0.03480095788836479, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.00005894530450833747, - "sigma_level": 92.1492706828521, - "cpk": 30.44918332024425, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.21353816986083984, - "mean_abs_diff": 0.032337937504053116, - "cosine_similarity": 0.9999272227287292, - "kl_divergence": 0.0005776968588551011, - "sigma_level": 96.96144440466122, - "cpk": 32.05918704076535, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.18230438232421875, - "mean_abs_diff": 0.029304036870598793, - "cosine_similarity": 0.9999476671218872, - "kl_divergence": 0.0005154337420336456, - "sigma_level": 107.47869468270946, - "cpk": 35.56376825825433, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.1879286766052246, - "mean_abs_diff": 0.029841218143701553, - "cosine_similarity": 0.9999344348907471, - "kl_divergence": 0.0001249257069489228, - "sigma_level": 104.64479499892016, - "cpk": 34.62137098670958, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.20756947994232178, - "mean_abs_diff": 0.029503241181373596, - "cosine_similarity": 0.9999513030052185, - "kl_divergence": 0.0007071416731914378, - "sigma_level": 106.82336906210507, - "cpk": 35.34515338559784, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.1847095489501953, - "mean_abs_diff": 0.028994187712669373, - "cosine_similarity": 0.9999274611473083, - "kl_divergence": 0.00020641965149885807, - "sigma_level": 107.64212517799307, - "cpk": 35.62062539406424, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.1772775650024414, - "mean_abs_diff": 0.029128575697541237, - "cosine_similarity": 0.9999530911445618, - "kl_divergence": 0.0001424306189745565, - "sigma_level": 107.36934004276407, - "cpk": 35.5291536851688, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.19080734252929688, - "mean_abs_diff": 0.024822678416967392, - "cosine_similarity": 0.9999489188194275, - "kl_divergence": 0.0004685865991592646, - "sigma_level": 125.75471839980924, - "cpk": 41.65810872208185, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.1714634895324707, - "mean_abs_diff": 0.02789604291319847, - "cosine_similarity": 0.9999091029167175, - "kl_divergence": 0.00015832598961693735, - "sigma_level": 112.04260562737294, - "cpk": 37.08707309790033, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.20160150527954102, - "mean_abs_diff": 0.03052336722612381, - "cosine_similarity": 0.9999374747276306, - "kl_divergence": 0.00035990273757939453, - "sigma_level": 103.43257387477989, - "cpk": 34.21443208863304, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.1712021827697754, - "mean_abs_diff": 0.026527106761932373, - "cosine_similarity": 0.9999428391456604, - "kl_divergence": 0.0004203473145415497, - "sigma_level": 117.54782949495923, - "cpk": 38.92275951309922, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.19522953033447266, - "mean_abs_diff": 0.028979429975152016, - "cosine_similarity": 0.9999217987060547, - "kl_divergence": 0.000012096107530428578, - "sigma_level": 108.97087819271881, - "cpk": 36.060466569746545, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.21483612060546875, - "mean_abs_diff": 0.03282041847705841, - "cosine_similarity": 0.999923586845398, - "kl_divergence": 0.0003550886859526683, - "sigma_level": 95.4968733707598, - "cpk": 31.571103844646682, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.19664621353149414, - "mean_abs_diff": 0.03151273727416992, - "cosine_similarity": 0.9999220967292786, - "kl_divergence": 0.0005721978732455467, - "sigma_level": 99.34248182400384, - "cpk": 32.853281147177626, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.23045682907104492, - "mean_abs_diff": 0.048200823366642, - "cosine_similarity": 0.9998908638954163, - "kl_divergence": 0.0011748968653306612, - "sigma_level": 69.28219447916557, - "cpk": 22.815776591509888, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2098684310913086, - "mean_abs_diff": 0.030793752521276474, - "cosine_similarity": 0.9999160766601562, - "kl_divergence": 0.00023351596473067916, - "sigma_level": 102.0201738769548, - "cpk": 33.74492596010622, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.25650787353515625, - "mean_abs_diff": 0.037612028419971466, - "cosine_similarity": 0.9999205470085144, - "kl_divergence": 0.0009010070401926103, - "sigma_level": 85.49162609615232, - "cpk": 28.229249242850933, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.19126582145690918, - "mean_abs_diff": 0.029047662392258644, - "cosine_similarity": 0.9999561905860901, - "kl_divergence": 0.0006576481104213344, - "sigma_level": 107.47869468270946, - "cpk": 35.566064491111156, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.20811530947685242, - "mean_abs_diff": 0.03056376241147518, - "cosine_similarity": 0.9999154210090637, - "kl_divergence": 0.0003487269693072701, - "sigma_level": 102.75772973693056, - "cpk": 33.990854675841675, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.22626805305480957, - "mean_abs_diff": 0.03359080106019974, - "cosine_similarity": 0.9998672008514404, - "kl_divergence": 0.0007366287035674455, - "sigma_level": 93.41415606507609, - "cpk": 30.876563993976326, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.8031983375549316, + "mean_abs_diff": 0.12188125401735306, + "cosine_similarity": 0.9984647035598755, + "kl_divergence": 0.011949143213562554, + "sigma_level": 26.330537211297408, + "cpk": 8.50941249591051, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.24585700035095215, + "mean_abs_diff": 0.03762512281537056, + "cosine_similarity": 0.9998586177825928, + "kl_divergence": 1.3277331549664009e-05, + "sigma_level": 84.0184221427533, + "cpk": 27.742707093261536, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.2099609375, + "mean_abs_diff": 0.033425211906433105, + "cosine_similarity": 0.9999147653579712, + "kl_divergence": 1.5090158264594303e-06, + "sigma_level": 94.57560609882762, + "cpk": 31.261767893356485, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.22704386711120605, + "mean_abs_diff": 0.0315629281103611, + "cosine_similarity": 0.999933660030365, + "kl_divergence": 3.7310540638633336e-05, + "sigma_level": 99.18469092152263, + "cpk": 32.80068370140717, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.2088146209716797, + "mean_abs_diff": 0.03077378123998642, + "cosine_similarity": 0.9999191164970398, + "kl_divergence": 1.4999781410245668e-06, + "sigma_level": 101.59603971600392, + "cpk": 33.60480538024553, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2547646760940552, + "mean_abs_diff": 0.03083600476384163, + "cosine_similarity": 0.9999075531959534, + "kl_divergence": 0.0002142816112921191, + "sigma_level": 101.00436248100775, + "cpk": 33.40857324344982, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.20438094437122345, + "mean_abs_diff": 0.031773317605257034, + "cosine_similarity": 0.9999481439590454, + "kl_divergence": 9.281385733172737e-07, + "sigma_level": 99.44615585236863, + "cpk": 32.88540742624628, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.18281269073486328, + "mean_abs_diff": 0.030860792845487595, + "cosine_similarity": 0.9999558925628662, + "kl_divergence": 2.0817027272510226e-06, + "sigma_level": 102.06207588666598, + "cpk": 33.75821558044542, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.28357791900634766, + "mean_abs_diff": 0.03390377387404442, + "cosine_similarity": 0.999890148639679, + "kl_divergence": 0.00045742262385036316, + "sigma_level": 92.77850864421463, + "cpk": 30.6640410832845, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2932462692260742, + "mean_abs_diff": 0.044488128274679184, + "cosine_similarity": 0.9998958706855774, + "kl_divergence": 0.0016326477262920424, + "sigma_level": 71.4951399913799, + "cpk": 23.566656250547247, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.25107574462890625, + "mean_abs_diff": 0.036512766033411026, + "cosine_similarity": 0.9999284148216248, + "kl_divergence": 0.0011823933043749269, + "sigma_level": 85.6818989695192, + "cpk": 28.299926062308696, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.25175952911376953, + "mean_abs_diff": 0.04251187667250633, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.000520927378713784, + "sigma_level": 75.90532316155377, + "cpk": 25.032867909098698, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.23739862442016602, + "mean_abs_diff": 0.035071682184934616, + "cosine_similarity": 0.9999188184738159, + "kl_divergence": 0.00016192173524096987, + "sigma_level": 91.2163777694204, + "cpk": 30.13886660554096, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.20715034008026123, + "mean_abs_diff": 0.031439900398254395, + "cosine_similarity": 0.9999179244041443, + "kl_divergence": 0.0007082983479816206, + "sigma_level": 100.4901828181864, + "cpk": 33.23344416116162, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.23068474233150482, + "mean_abs_diff": 0.03183136135339737, + "cosine_similarity": 0.9999409914016724, + "kl_divergence": 0.0007547449349859649, + "sigma_level": 98.39571148132515, + "cpk": 32.53756470645949, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23305463790893555, + "mean_abs_diff": 0.028933987021446228, + "cosine_similarity": 0.9998893141746521, + "kl_divergence": 0.0003845604489353084, + "sigma_level": 107.47455288493924, + "cpk": 35.565712018454036, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2608985900878906, + "mean_abs_diff": 0.04424131289124489, + "cosine_similarity": 0.9999105930328369, + "kl_divergence": 0.0007788479711618704, + "sigma_level": 73.24530032745929, + "cpk": 24.14506108835307, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.2301793098449707, + "mean_abs_diff": 0.0324413925409317, + "cosine_similarity": 0.9999143481254578, + "kl_divergence": 2.250089535157956e-05, + "sigma_level": 96.25744820928855, + "cpk": 31.825588931234023, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.19923532009124756, + "mean_abs_diff": 0.03906163200736046, + "cosine_similarity": 0.999877393245697, + "kl_divergence": 0.0005163370630241004, + "sigma_level": 83.7633797367138, + "cpk": 27.648465385990573, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 5942, + "gpu_argmax": 5942, + "max_abs_diff": 0.17966842651367188, + "mean_abs_diff": 0.02848205156624317, + "cosine_similarity": 0.9999198317527771, + "kl_divergence": 0.0008812988738490747, + "sigma_level": 109.6664749254824, + "cpk": 36.29519779233449, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.29142284393310547, + "mean_abs_diff": 0.043598152697086334, + "cosine_similarity": 0.9998776316642761, + "kl_divergence": 0.0010248388343692954, + "sigma_level": 73.4393195777277, + "cpk": 24.21295497016591, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.34327876567840576, + "mean_abs_diff": 0.0488663986325264, + "cosine_similarity": 0.9999039173126221, + "kl_divergence": 0.0018092925711578334, + "sigma_level": 64.72170503736106, + "cpk": 21.310341959242646, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3739614486694336, + "mean_abs_diff": 0.04563745856285095, + "cosine_similarity": 0.9998410940170288, + "kl_divergence": 0.0013028657116986625, + "sigma_level": 68.52047136061793, + "cpk": 22.579565439170377, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.25923848152160645, + "mean_abs_diff": 0.04782666265964508, + "cosine_similarity": 0.9998435378074646, + "kl_divergence": 0.00017440859928140556, + "sigma_level": 68.0917989073902, + "cpk": 22.425882677777388, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.20351624488830566, + "mean_abs_diff": 0.030075596645474434, + "cosine_similarity": 0.9998764991760254, + "kl_divergence": 0.0005101532884886966, + "sigma_level": 103.98703355346954, + "cpk": 34.40172184469705, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.20961999893188477, + "mean_abs_diff": 0.0313001312315464, + "cosine_similarity": 0.9998053312301636, + "kl_divergence": 2.8302109163839864e-05, + "sigma_level": 102.59202935793, + "cpk": 33.929747787458844, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19363737106323242, + "mean_abs_diff": 0.033073790371418, + "cosine_similarity": 0.9998298287391663, + "kl_divergence": 0.0005171189703267121, + "sigma_level": 96.9427542695765, + "cpk": 32.04706272879717, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.20902681350708008, + "mean_abs_diff": 0.03569749742746353, + "cosine_similarity": 0.9997431039810181, + "kl_divergence": 0.0001146219937046406, + "sigma_level": 90.59358669300643, + "cpk": 29.92836520367562, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.20511221885681152, + "mean_abs_diff": 0.03204226866364479, + "cosine_similarity": 0.9998698830604553, + "kl_divergence": 0.0009168103274381441, + "sigma_level": 97.91782700745156, + "cpk": 32.3778165591561, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.20564067363739014, + "mean_abs_diff": 0.03282441571354866, + "cosine_similarity": 0.9998794794082642, + "kl_divergence": 3.538160864634364e-05, + "sigma_level": 96.38664467254054, + "cpk": 31.865228616349718, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2244204878807068, + "mean_abs_diff": 0.048710912466049194, + "cosine_similarity": 0.9996637105941772, + "kl_divergence": 3.898172129763024e-06, + "sigma_level": 69.19204623397793, + "cpk": 22.7831481023718, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19256067276000977, + "mean_abs_diff": 0.038800157606601715, + "cosine_similarity": 0.9998534321784973, + "kl_divergence": 0.00014612402448690114, + "sigma_level": 83.81763111733832, + "cpk": 27.668198930982378, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.24395322799682617, + "mean_abs_diff": 0.035089071840047836, + "cosine_similarity": 0.9998824596405029, + "kl_divergence": 0.0007183325325718559, + "sigma_level": 90.07186375980564, + "cpk": 29.76057641174898, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.22237825393676758, + "mean_abs_diff": 0.030163198709487915, + "cosine_similarity": 0.9998973608016968, + "kl_divergence": 0.0007083169726316857, + "sigma_level": 103.65308835461354, + "cpk": 34.290487059796824, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.19781363010406494, + "mean_abs_diff": 0.028445789590477943, + "cosine_similarity": 0.9998837113380432, + "kl_divergence": 9.986173050612662e-06, + "sigma_level": 109.84427252512704, + "cpk": 36.35437358637827, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.21171927452087402, + "mean_abs_diff": 0.0331043042242527, + "cosine_similarity": 0.999922513961792, + "kl_divergence": 0.0005099449753999734, + "sigma_level": 95.18447747645524, + "cpk": 31.465574500501155, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.2000875473022461, + "mean_abs_diff": 0.028575299307703972, + "cosine_similarity": 0.9998475313186646, + "kl_divergence": 0.001018203258176856, + "sigma_level": 109.12304284617422, + "cpk": 36.11449564783334, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.18700814247131348, + "mean_abs_diff": 0.026265908032655716, + "cosine_similarity": 0.9998998045921326, + "kl_divergence": 0.00046225863274036456, + "sigma_level": 119.02215463941639, + "cpk": 39.413532799171506, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.22919845581054688, + "mean_abs_diff": 0.04531732201576233, + "cosine_similarity": 0.9998840689659119, + "kl_divergence": 0.0008305399123272189, + "sigma_level": 72.06274536754663, + "cpk": 23.74877423608546, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23381578922271729, + "mean_abs_diff": 0.030515603721141815, + "cosine_similarity": 0.9999016523361206, + "kl_divergence": 0.0005923224650756628, + "sigma_level": 103.29603430637736, + "cpk": 34.169333031387545, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.19087600708007812, + "mean_abs_diff": 0.030218342319130898, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 0.00045970970727517754, + "sigma_level": 104.27458896076459, + "cpk": 34.49561255155461, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2747793197631836, + "mean_abs_diff": 0.03750928118824959, + "cosine_similarity": 0.9999193549156189, + "kl_divergence": 0.00287010815315867, + "sigma_level": 84.41037393510442, + "cpk": 27.872943607440046, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.18088853359222412, + "mean_abs_diff": 0.029429737478494644, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 0.0002790777073941985, + "sigma_level": 106.92372344549646, + "cpk": 35.37901305589681, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.200392484664917, + "mean_abs_diff": 0.03397694602608681, + "cosine_similarity": 0.9998542070388794, + "kl_divergence": 0.0003240177982787965, + "sigma_level": 94.48294788690083, + "cpk": 31.226795793905374, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.29866647720336914, + "mean_abs_diff": 0.045978154987096786, + "cosine_similarity": 0.9998794198036194, + "kl_divergence": 0.0009558542296503378, + "sigma_level": 68.50529301922718, + "cpk": 22.572618758086186, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.21673345565795898, + "mean_abs_diff": 0.03529226407408714, + "cosine_similarity": 0.9999400973320007, + "kl_divergence": 0.002097586731727194, + "sigma_level": 90.45730879559231, + "cpk": 29.886399329410327, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.22467482089996338, + "mean_abs_diff": 0.03361072018742561, + "cosine_similarity": 0.999942421913147, + "kl_divergence": 0.0008436583035788792, + "sigma_level": 93.8695872897644, + "cpk": 31.026943727212682, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.22781848907470703, + "mean_abs_diff": 0.030904922634363174, + "cosine_similarity": 0.9999483227729797, + "kl_divergence": 0.0008333849906099469, + "sigma_level": 102.17588221636045, + "cpk": 33.795482594203946, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1783003807067871, + "mean_abs_diff": 0.029902499169111252, + "cosine_similarity": 0.9999409317970276, + "kl_divergence": 0.0003094355721386634, + "sigma_level": 106.19563387482063, + "cpk": 35.13391838713145, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2627032399177551, + "mean_abs_diff": 0.03310393542051315, + "cosine_similarity": 0.9999225735664368, + "kl_divergence": 0.00015295545004923058, + "sigma_level": 94.89749613646042, + "cpk": 31.37070866351432, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.19065427780151367, + "mean_abs_diff": 0.028850441798567772, + "cosine_similarity": 0.9999325275421143, + "kl_divergence": 0.00038527130067871274, + "sigma_level": 108.36687717839267, + "cpk": 35.861756369220245, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.23432064056396484, + "mean_abs_diff": 0.037413571029901505, + "cosine_similarity": 0.9999155402183533, + "kl_divergence": 0.0002889698267149911, + "sigma_level": 85.30887203134782, + "cpk": 28.17031488184714, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.18627315759658813, + "mean_abs_diff": 0.02875906601548195, + "cosine_similarity": 0.9999480843544006, + "kl_divergence": 0.0007701578612177225, + "sigma_level": 108.96690810906699, + "cpk": 36.06115382770469, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.22917795181274414, + "mean_abs_diff": 0.03641311824321747, + "cosine_similarity": 0.9999172687530518, + "kl_divergence": 0.0005103069102512929, + "sigma_level": 87.72677177564587, + "cpk": 28.97605681573509, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.24593007564544678, + "mean_abs_diff": 0.03212062641978264, + "cosine_similarity": 0.9998915791511536, + "kl_divergence": 3.151487057192769e-05, + "sigma_level": 97.4480464319994, + "cpk": 32.221841119434806, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.19705867767333984, + "mean_abs_diff": 0.0312761552631855, + "cosine_similarity": 0.9999309778213501, + "kl_divergence": 0.0001150733661477417, + "sigma_level": 100.417536631287, + "cpk": 33.210789338193436, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.1876506805419922, + "mean_abs_diff": 0.031015293672680855, + "cosine_similarity": 0.9999198913574219, + "kl_divergence": 0.0007250441446097092, + "sigma_level": 101.70368336368918, + "cpk": 33.638363653969876, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.19601619243621826, + "mean_abs_diff": 0.03426588699221611, + "cosine_similarity": 0.999900221824646, + "kl_divergence": 0.00023789191270330738, + "sigma_level": 93.12063513483277, + "cpk": 30.77430661492979, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.20610380172729492, + "mean_abs_diff": 0.03480095788836479, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 5.894530450833747e-05, + "sigma_level": 92.1492706828521, + "cpk": 30.44918332024425, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.21353816986083984, + "mean_abs_diff": 0.032337937504053116, + "cosine_similarity": 0.9999272227287292, + "kl_divergence": 0.0005776968588551011, + "sigma_level": 96.96144440466122, + "cpk": 32.05918704076535, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.18230438232421875, + "mean_abs_diff": 0.029304036870598793, + "cosine_similarity": 0.9999476671218872, + "kl_divergence": 0.0005154337420336456, + "sigma_level": 107.47869468270946, + "cpk": 35.56376825825433, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.1879286766052246, + "mean_abs_diff": 0.029841218143701553, + "cosine_similarity": 0.9999344348907471, + "kl_divergence": 0.0001249257069489228, + "sigma_level": 104.64479499892016, + "cpk": 34.62137098670958, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.20756947994232178, + "mean_abs_diff": 0.029503241181373596, + "cosine_similarity": 0.9999513030052185, + "kl_divergence": 0.0007071416731914378, + "sigma_level": 106.82336906210507, + "cpk": 35.34515338559784, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.1847095489501953, + "mean_abs_diff": 0.028994187712669373, + "cosine_similarity": 0.9999274611473083, + "kl_divergence": 0.00020641965149885807, + "sigma_level": 107.64212517799307, + "cpk": 35.62062539406424, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.1772775650024414, + "mean_abs_diff": 0.029128575697541237, + "cosine_similarity": 0.9999530911445618, + "kl_divergence": 0.0001424306189745565, + "sigma_level": 107.36934004276407, + "cpk": 35.5291536851688, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.19080734252929688, + "mean_abs_diff": 0.024822678416967392, + "cosine_similarity": 0.9999489188194275, + "kl_divergence": 0.0004685865991592646, + "sigma_level": 125.75471839980924, + "cpk": 41.65810872208185, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.1714634895324707, + "mean_abs_diff": 0.02789604291319847, + "cosine_similarity": 0.9999091029167175, + "kl_divergence": 0.00015832598961693735, + "sigma_level": 112.04260562737294, + "cpk": 37.08707309790033, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.20160150527954102, + "mean_abs_diff": 0.03052336722612381, + "cosine_similarity": 0.9999374747276306, + "kl_divergence": 0.00035990273757939453, + "sigma_level": 103.43257387477989, + "cpk": 34.21443208863304, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.1712021827697754, + "mean_abs_diff": 0.026527106761932373, + "cosine_similarity": 0.9999428391456604, + "kl_divergence": 0.0004203473145415497, + "sigma_level": 117.54782949495923, + "cpk": 38.92275951309922, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.19522953033447266, + "mean_abs_diff": 0.028979429975152016, + "cosine_similarity": 0.9999217987060547, + "kl_divergence": 1.2096107530428578e-05, + "sigma_level": 108.97087819271881, + "cpk": 36.060466569746545, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.21483612060546875, + "mean_abs_diff": 0.03282041847705841, + "cosine_similarity": 0.999923586845398, + "kl_divergence": 0.0003550886859526683, + "sigma_level": 95.4968733707598, + "cpk": 31.571103844646682, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.19664621353149414, + "mean_abs_diff": 0.03151273727416992, + "cosine_similarity": 0.9999220967292786, + "kl_divergence": 0.0005721978732455467, + "sigma_level": 99.34248182400384, + "cpk": 32.853281147177626, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.23045682907104492, + "mean_abs_diff": 0.048200823366642, + "cosine_similarity": 0.9998908638954163, + "kl_divergence": 0.0011748968653306612, + "sigma_level": 69.28219447916557, + "cpk": 22.815776591509888, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2098684310913086, + "mean_abs_diff": 0.030793752521276474, + "cosine_similarity": 0.9999160766601562, + "kl_divergence": 0.00023351596473067916, + "sigma_level": 102.0201738769548, + "cpk": 33.74492596010622, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.25650787353515625, + "mean_abs_diff": 0.037612028419971466, + "cosine_similarity": 0.9999205470085144, + "kl_divergence": 0.0009010070401926103, + "sigma_level": 85.49162609615232, + "cpk": 28.229249242850933, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.19126582145690918, + "mean_abs_diff": 0.029047662392258644, + "cosine_similarity": 0.9999561905860901, + "kl_divergence": 0.0006576481104213344, + "sigma_level": 107.47869468270946, + "cpk": 35.566064491111156, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.20811530947685242, + "mean_abs_diff": 0.03056376241147518, + "cosine_similarity": 0.9999154210090637, + "kl_divergence": 0.0003487269693072701, + "sigma_level": 102.75772973693056, + "cpk": 33.990854675841675, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.22626805305480957, + "mean_abs_diff": 0.03359080106019974, + "cosine_similarity": 0.9998672008514404, + "kl_divergence": 0.0007366287035674455, + "sigma_level": 93.41415606507609, + "cpk": 30.876563993976326, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json index ce968eac37..20ddb8ff92 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1068 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-1.5b-instruct-q4_k_m", + "file": "~/models/qwen2.5-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M", + "model_sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.66.0", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "342f4199a612c842", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md \u00a7'2026-09-09 \u2014 the reporter's exact file, measured'", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`.", + "model_sha256_basis": "evidence/parity/l0-1/lambda/RECORD.md \u2014 fetched from Qwen/Qwen2.5-1.5B-Instruct-GGUF, 1117320736 bytes, verified against the HF LFS oid at measurement time" + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9978237152099609, + "min_cosine_position": 4, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "~/models/qwen2.5-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 2701, - "gpu_argmax": 2701, - "max_abs_diff": 0.31842291355133057, - "mean_abs_diff": 0.05168678238987923, - "cosine_similarity": 0.9998032450675964, - "kl_divergence": 0.0010041628555806199, - "sigma_level": 61.438975798323675, - "cpk": 20.215026685079142, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.6011629104614258, - "mean_abs_diff": 0.09295977652072906, - "cosine_similarity": 0.9993610382080078, - "kl_divergence": 0.00038596963427606305, - "sigma_level": 33.920919979632444, - "cpk": 11.044199898153801, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 1.074141263961792, - "mean_abs_diff": 0.1629197895526886, - "cosine_similarity": 0.998677670955658, - "kl_divergence": 0.00015600067604519665, - "sigma_level": 19.209160557495856, - "cpk": 6.142257486206033, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.8302667140960693, - "mean_abs_diff": 0.14872857928276062, - "cosine_similarity": 0.9985498189926147, - "kl_divergence": 0.00035117984368604677, - "sigma_level": 21.646905131259768, - "cpk": 6.947342256583176, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 1.2988052368164062, - "mean_abs_diff": 0.2101205289363861, - "cosine_similarity": 0.9978237152099609, - "kl_divergence": 0.0007265903462690505, - "sigma_level": 15.178418190254549, - "cpk": 4.793697958538687, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 1.0872032642364502, - "mean_abs_diff": 0.16018898785114288, - "cosine_similarity": 0.9980590343475342, - "kl_divergence": 0.013828814951459963, - "sigma_level": 19.748898118047645, - "cpk": 6.3193363726237655, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.9903631210327148, - "mean_abs_diff": 0.17371346056461334, - "cosine_similarity": 0.9984952807426453, - "kl_divergence": 0.0038416856287773597, - "sigma_level": 18.438460185295785, - "cpk": 5.879236001242714, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.7914443016052246, - "mean_abs_diff": 0.13904337584972382, - "cosine_similarity": 0.9994173645973206, - "kl_divergence": 0.0011267766165267705, - "sigma_level": 23.019298841074793, - "cpk": 7.406376195311875, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 624, - "gpu_argmax": 624, - "max_abs_diff": 0.7683401107788086, - "mean_abs_diff": 0.11628174781799316, - "cosine_similarity": 0.9988746643066406, - "kl_divergence": 0.0021637816443614774, - "sigma_level": 27.418492188242922, - "cpk": 8.873808213232401, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.5380454063415527, - "mean_abs_diff": 0.10169024765491486, - "cosine_similarity": 0.999545693397522, - "kl_divergence": 0.004308393425260426, - "sigma_level": 31.753008418995396, - "cpk": 10.315255198838779, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 3974, - "gpu_argmax": 3974, - "max_abs_diff": 0.4585554599761963, - "mean_abs_diff": 0.06840988248586655, - "cosine_similarity": 0.9997379183769226, - "kl_divergence": 0.003921000934134561, - "sigma_level": 46.307180649881325, - "cpk": 15.17173781775126, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.5177440643310547, - "mean_abs_diff": 0.06941164284944534, - "cosine_similarity": 0.9996582865715027, - "kl_divergence": 0.0020656723212676966, - "sigma_level": 45.76503932690633, - "cpk": 14.99029422857294, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.420398473739624, - "mean_abs_diff": 0.06813345104455948, - "cosine_similarity": 0.9997531175613403, - "kl_divergence": 0.0012232872752691486, - "sigma_level": 46.26818246767472, - "cpk": 15.160026577134735, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.4519714117050171, - "mean_abs_diff": 0.0760267823934555, - "cosine_similarity": 0.9996703267097473, - "kl_divergence": 0.001647045695020044, - "sigma_level": 41.99132346083567, - "cpk": 13.731069052681043, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 9293, - "gpu_argmax": 9293, - "max_abs_diff": 0.44501709938049316, - "mean_abs_diff": 0.08212687820196152, - "cosine_similarity": 0.999630331993103, - "kl_divergence": 0.006501142529339967, - "sigma_level": 38.93182174829522, - "cpk": 12.710828167523179, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 369, - "gpu_argmax": 369, - "max_abs_diff": 0.46442174911499023, - "mean_abs_diff": 0.08806511759757996, - "cosine_similarity": 0.9996541738510132, - "kl_divergence": 0.0013393907643138331, - "sigma_level": 36.201662750395194, - "cpk": 11.801545609519941, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.41184115409851074, - "mean_abs_diff": 0.0735999047756195, - "cosine_similarity": 0.9997572898864746, - "kl_divergence": 0.0020352302058443735, - "sigma_level": 42.657347277985636, - "cpk": 13.957484367858523, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.39466023445129395, - "mean_abs_diff": 0.07817857712507248, - "cosine_similarity": 0.999573826789856, - "kl_divergence": 0.00016666892717610718, - "sigma_level": 41.49852594567503, - "cpk": 13.562484005956613, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.5057002902030945, - "mean_abs_diff": 0.07810698449611664, - "cosine_similarity": 0.9997146129608154, - "kl_divergence": 0.0018880510517696614, - "sigma_level": 40.865303812203365, - "cpk": 13.355779133127049, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 14310, - "gpu_argmax": 14310, - "max_abs_diff": 0.4656808376312256, - "mean_abs_diff": 0.07576054334640503, - "cosine_similarity": 0.9997501373291016, - "kl_divergence": 0.0019064405359464263, - "sigma_level": 42.13443236727582, - "cpk": 13.778800164947176, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.3974335193634033, - "mean_abs_diff": 0.07983855158090591, - "cosine_similarity": 0.999765157699585, - "kl_divergence": 0.003539420801547969, - "sigma_level": 40.48393264839967, - "cpk": 13.225296004054293, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.5480650663375854, - "mean_abs_diff": 0.09393318742513657, - "cosine_similarity": 0.999376118183136, - "kl_divergence": 0.0038084510052187473, - "sigma_level": 33.93247368937331, - "cpk": 11.045209112219235, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.5794413089752197, - "mean_abs_diff": 0.08977903425693512, - "cosine_similarity": 0.9995113015174866, - "kl_divergence": 0.002239943144359716, - "sigma_level": 34.99617146302287, - "cpk": 11.40356361462093, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.4805694818496704, - "mean_abs_diff": 0.07361047714948654, - "cosine_similarity": 0.9997532367706299, - "kl_divergence": 0.0017476027456557173, - "sigma_level": 43.46233730592578, - "cpk": 14.220838819715166, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.575049638748169, - "mean_abs_diff": 0.08266887068748474, - "cosine_similarity": 0.9996919631958008, - "kl_divergence": 0.005820226039888798, - "sigma_level": 38.466205868760724, - "cpk": 12.55707213968668, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4894747734069824, - "mean_abs_diff": 0.07466553151607513, - "cosine_similarity": 0.9998354911804199, - "kl_divergence": 0.001684058784819267, - "sigma_level": 42.247815077849495, - "cpk": 13.819733728601458, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.39302289485931396, - "mean_abs_diff": 0.0665874183177948, - "cosine_similarity": 0.9998313784599304, - "kl_divergence": 0.0034372790646476988, - "sigma_level": 47.353553428886435, - "cpk": 15.521755237045204, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.41148829460144043, - "mean_abs_diff": 0.0705053061246872, - "cosine_similarity": 0.9995219707489014, - "kl_divergence": 0.000481547916431422, - "sigma_level": 44.680539884015715, - "cpk": 14.63099536614367, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.4649146795272827, - "mean_abs_diff": 0.07782384008169174, - "cosine_similarity": 0.9995982646942139, - "kl_divergence": 0.0015583181127582582, - "sigma_level": 40.227398043234544, - "cpk": 13.148245131726577, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.47322654724121094, - "mean_abs_diff": 0.087704598903656, - "cosine_similarity": 0.9995580315589905, - "kl_divergence": 0.0022127799633092564, - "sigma_level": 36.708986647160415, - "cpk": 11.968033303216066, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.430513858795166, - "mean_abs_diff": 0.08133339881896973, - "cosine_similarity": 0.9997889995574951, - "kl_divergence": 0.00018704778171310847, - "sigma_level": 39.01572660971666, - "cpk": 12.740802065525553, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.45681631565093994, - "mean_abs_diff": 0.08105611801147461, - "cosine_similarity": 0.9997667074203491, - "kl_divergence": 0.003126264705790457, - "sigma_level": 39.84984885704753, - "cpk": 13.01411011470782, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.36823856830596924, - "mean_abs_diff": 0.061732012778520584, - "cosine_similarity": 0.9998225569725037, - "kl_divergence": 0.0028589001523262048, - "sigma_level": 50.88439404151175, - "cpk": 16.699698341904092, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 1075, - "gpu_argmax": 1075, - "max_abs_diff": 0.45197248458862305, - "mean_abs_diff": 0.06759438663721085, - "cosine_similarity": 0.9998044967651367, - "kl_divergence": 0.0044328146591125035, - "sigma_level": 46.359707613196264, - "cpk": 15.192097870999218, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.4539768099784851, - "mean_abs_diff": 0.07912351936101913, - "cosine_similarity": 0.9997790455818176, - "kl_divergence": 0.00013784763067440514, - "sigma_level": 40.56873603366942, - "cpk": 13.255416913638795, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.38960933685302734, - "mean_abs_diff": 0.06396492570638657, - "cosine_similarity": 0.9998103380203247, - "kl_divergence": 0.0029847960362235373, - "sigma_level": 49.1493031479779, - "cpk": 16.121115088960877, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.40729308128356934, - "mean_abs_diff": 0.06924924999475479, - "cosine_similarity": 0.9996633529663086, - "kl_divergence": 0.005449064420257557, - "sigma_level": 45.52429008065333, - "cpk": 14.912053114832036, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.36036384105682373, - "mean_abs_diff": 0.05879916250705719, - "cosine_similarity": 0.9997984170913696, - "kl_divergence": 0.00044588987994617586, - "sigma_level": 53.598492891250764, - "cpk": 17.60353542261309, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 8003, - "gpu_argmax": 17944, - "max_abs_diff": 0.4275398254394531, - "mean_abs_diff": 0.06924005597829819, - "cosine_similarity": 0.9997360110282898, - "kl_divergence": 0.0030625878432387578, - "sigma_level": 45.48962434668141, - "cpk": 14.900732770877472, - "verdict": "WarnArgmax" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 16293, - "gpu_argmax": 16293, - "max_abs_diff": 0.4567922353744507, - "mean_abs_diff": 0.07736830413341522, - "cosine_similarity": 0.9996188879013062, - "kl_divergence": 0.00627417830634242, - "sigma_level": 40.883418450180905, - "cpk": 13.36421608733803, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.42419886589050293, - "mean_abs_diff": 0.07781759649515152, - "cosine_similarity": 0.9997288584709167, - "kl_divergence": 0.00445967416527253, - "sigma_level": 40.85204776562423, - "cpk": 13.352431907789242, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2849, - "gpu_argmax": 2849, - "max_abs_diff": 0.41576290130615234, - "mean_abs_diff": 0.06628850847482681, - "cosine_similarity": 0.9997788071632385, - "kl_divergence": 0.002364125347954264, - "sigma_level": 47.590142022032, - "cpk": 15.600490712948526, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 15409, - "gpu_argmax": 15409, - "max_abs_diff": 0.4992527961730957, - "mean_abs_diff": 0.08212177455425262, - "cosine_similarity": 0.9995926022529602, - "kl_divergence": 0.0024074922898179433, - "sigma_level": 39.21766934638157, - "cpk": 12.804171065409959, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.5810892581939697, - "mean_abs_diff": 0.0682300254702568, - "cosine_similarity": 0.999836266040802, - "kl_divergence": 0.002712155435780897, - "sigma_level": 46.05452141964385, - "cpk": 15.08964870909105, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.43909335136413574, - "mean_abs_diff": 0.07037733495235443, - "cosine_similarity": 0.9998226165771484, - "kl_divergence": 0.0024886335454403913, - "sigma_level": 44.62151623300525, - "cpk": 14.612143461500741, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.5455688238143921, - "mean_abs_diff": 0.08337622880935669, - "cosine_similarity": 0.9997198581695557, - "kl_divergence": 0.004219288243595902, - "sigma_level": 38.8716398899516, - "cpk": 12.68713240151224, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.4702770709991455, - "mean_abs_diff": 0.07033935934305191, - "cosine_similarity": 0.9997967481613159, - "kl_divergence": 0.002269168837427208, - "sigma_level": 44.76284014389586, - "cpk": 14.658564256457195, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.47287213802337646, - "mean_abs_diff": 0.08464169502258301, - "cosine_similarity": 0.9996572136878967, - "kl_divergence": 0.0032647584277852933, - "sigma_level": 38.09939719641655, - "cpk": 12.431065935635237, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.403623104095459, - "mean_abs_diff": 0.08138824999332428, - "cosine_similarity": 0.9997397065162659, - "kl_divergence": 0.002429769070231016, - "sigma_level": 39.845518962009926, - "cpk": 13.011593232470492, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.41884738206863403, - "mean_abs_diff": 0.07357224076986313, - "cosine_similarity": 0.9997681975364685, - "kl_divergence": 0.0026198766285110034, - "sigma_level": 43.19010015433112, - "cpk": 14.131900680824627, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.38974857330322266, - "mean_abs_diff": 0.06174059584736824, - "cosine_similarity": 0.9997618794441223, - "kl_divergence": 0.0024988794185040867, - "sigma_level": 51.257036123035775, - "cpk": 16.8219587117114, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.4303196668624878, - "mean_abs_diff": 0.07956542819738388, - "cosine_similarity": 0.9997139573097229, - "kl_divergence": 0.000753836734309782, - "sigma_level": 40.610606016595504, - "cpk": 13.267601984109694, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.39939403533935547, - "mean_abs_diff": 0.06406113505363464, - "cosine_similarity": 0.9996994137763977, - "kl_divergence": 0.0021454389459888124, - "sigma_level": 49.09465897252818, - "cpk": 16.102798025938455, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4145479202270508, - "mean_abs_diff": 0.06807403266429901, - "cosine_similarity": 0.9997435808181763, - "kl_divergence": 0.0013922520524832468, - "sigma_level": 46.44678116489198, - "cpk": 15.218775413449793, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.39621949195861816, - "mean_abs_diff": 0.06436076015233994, - "cosine_similarity": 0.9998461008071899, - "kl_divergence": 0.0007129833951081995, - "sigma_level": 48.813617709656256, - "cpk": 16.009399108087152, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.5128415822982788, - "mean_abs_diff": 0.07186087220907211, - "cosine_similarity": 0.9997901320457458, - "kl_divergence": 0.0010620789511583434, - "sigma_level": 44.16569226501575, - "cpk": 14.457415324348458, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.4191385507583618, - "mean_abs_diff": 0.0679415762424469, - "cosine_similarity": 0.9998086094856262, - "kl_divergence": 0.0027788771123882566, - "sigma_level": 46.44338193961324, - "cpk": 15.218174265287137, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.36243438720703125, - "mean_abs_diff": 0.06031492352485657, - "cosine_similarity": 0.9998459815979004, - "kl_divergence": 0.0011979330012718125, - "sigma_level": 52.57512210986416, - "cpk": 17.26078533084085, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.4538910984992981, - "mean_abs_diff": 0.06918986886739731, - "cosine_similarity": 0.9997843503952026, - "kl_divergence": 0.0007243342259276751, - "sigma_level": 45.90852515083764, - "cpk": 15.038141314022376, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.44234466552734375, - "mean_abs_diff": 0.0634784922003746, - "cosine_similarity": 0.9998288154602051, - "kl_divergence": 0.0007674909766671851, - "sigma_level": 49.43904155443516, - "cpk": 16.21815420033612, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.351959228515625, - "mean_abs_diff": 0.05642838776111603, - "cosine_similarity": 0.999832808971405, - "kl_divergence": 0.0019997153002613145, - "sigma_level": 55.90082537578916, - "cpk": 18.370742337723765, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.38826751708984375, - "mean_abs_diff": 0.0642109364271164, - "cosine_similarity": 0.9998274445533752, - "kl_divergence": 0.0006867424058280134, - "sigma_level": 49.15437011428163, - "cpk": 16.121769360216955, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.37309741973876953, - "mean_abs_diff": 0.06227777898311615, - "cosine_similarity": 0.9998220205307007, - "kl_divergence": 0.0021385287649098024, - "sigma_level": 50.534553708367916, - "cpk": 16.58258625555096, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 18589, - "gpu_argmax": 18589, - "max_abs_diff": 0.3214547634124756, - "mean_abs_diff": 0.053083546459674835, - "cosine_similarity": 0.9998363256454468, - "kl_divergence": 0.000752778125991802, - "sigma_level": 59.39349988162695, - "cpk": 19.535098493011564, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.36084842681884766, - "mean_abs_diff": 0.061356306076049805, - "cosine_similarity": 0.9998142123222351, - "kl_divergence": 0.003312728169157689, - "sigma_level": 51.05759445120621, - "cpk": 16.75813936768082, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 7354, - "gpu_argmax": 7354, - "max_abs_diff": 0.44296157360076904, - "mean_abs_diff": 0.06914495676755905, - "cosine_similarity": 0.9997162818908691, - "kl_divergence": 0.0032034377972692007, - "sigma_level": 46.07774351870567, - "cpk": 15.093744207606258, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.4447762966156006, - "mean_abs_diff": 0.07214333862066269, - "cosine_similarity": 0.999789834022522, - "kl_divergence": 0.002128935912980073, - "sigma_level": 44.064428985081875, - "cpk": 14.42323007657755, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1033, - "gpu_argmax": 1033, - "max_abs_diff": 0.43563520908355713, - "mean_abs_diff": 0.07464952021837234, - "cosine_similarity": 0.9997342228889465, - "kl_divergence": 0.0023521526351508127, - "sigma_level": 42.89405481276757, - "cpk": 14.031183219923053, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.41919422149658203, - "mean_abs_diff": 0.07205758988857269, - "cosine_similarity": 0.9996944665908813, - "kl_divergence": 0.0015906886315562518, - "sigma_level": 44.11487020371673, - "cpk": 14.44005579914499, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.4340832233428955, - "mean_abs_diff": 0.06832505017518997, - "cosine_similarity": 0.9997145533561707, - "kl_divergence": 4.531155001990606e-05, - "sigma_level": 46.65343302362604, - "cpk": 15.285511161860004, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.5156793594360352, - "mean_abs_diff": 0.07876264303922653, - "cosine_similarity": 0.9997888803482056, - "kl_divergence": 0.002400534868525818, - "sigma_level": 40.82280172433786, - "cpk": 13.339657928106357, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.49131178855895996, - "mean_abs_diff": 0.06945641338825226, - "cosine_similarity": 0.9997475743293762, - "kl_divergence": 0.0013757893037735654, - "sigma_level": 45.35377881058699, - "cpk": 14.855417036046704, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 2390, - "gpu_argmax": 2390, - "max_abs_diff": 0.5032455325126648, - "mean_abs_diff": 0.07041440159082413, - "cosine_similarity": 0.9997410178184509, - "kl_divergence": 0.0025526453726194957, - "sigma_level": 44.967915760286665, - "cpk": 14.72543951350829, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.47355103492736816, - "mean_abs_diff": 0.08038556575775146, - "cosine_similarity": 0.999701976776123, - "kl_divergence": 0.0016015019282984591, - "sigma_level": 39.61497617254028, - "cpk": 12.939619368170968, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.4618046283721924, - "mean_abs_diff": 0.07349393516778946, - "cosine_similarity": 0.999594509601593, - "kl_divergence": 0.0007789350432452571, - "sigma_level": 43.43102772649833, - "cpk": 14.211016147499297, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 1865, - "max_abs_diff": 0.40026283264160156, - "mean_abs_diff": 0.06240736320614815, - "cosine_similarity": 0.9997082948684692, - "kl_divergence": 0.002672701304671866, - "sigma_level": 50.514450324264075, - "cpk": 16.57544397070925, - "verdict": "WarnArgmax" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.38044023513793945, - "mean_abs_diff": 0.06645609438419342, - "cosine_similarity": 0.9998005628585815, - "kl_divergence": 0.000939163318030381, - "sigma_level": 47.37136548013599, - "cpk": 15.528112165423995, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.4195518493652344, - "mean_abs_diff": 0.06238096207380295, - "cosine_similarity": 0.9997071623802185, - "kl_divergence": 0.0019081753538363014, - "sigma_level": 50.68498889665763, - "cpk": 16.631514768046415, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 2701, + "gpu_argmax": 2701, + "max_abs_diff": 0.31842291355133057, + "mean_abs_diff": 0.05168678238987923, + "cosine_similarity": 0.9998032450675964, + "kl_divergence": 0.0010041628555806199, + "sigma_level": 61.438975798323675, + "cpk": 20.215026685079142, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.6011629104614258, + "mean_abs_diff": 0.09295977652072906, + "cosine_similarity": 0.9993610382080078, + "kl_divergence": 0.00038596963427606305, + "sigma_level": 33.920919979632444, + "cpk": 11.044199898153801, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 1.074141263961792, + "mean_abs_diff": 0.1629197895526886, + "cosine_similarity": 0.998677670955658, + "kl_divergence": 0.00015600067604519665, + "sigma_level": 19.209160557495856, + "cpk": 6.142257486206033, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.8302667140960693, + "mean_abs_diff": 0.14872857928276062, + "cosine_similarity": 0.9985498189926147, + "kl_divergence": 0.00035117984368604677, + "sigma_level": 21.646905131259768, + "cpk": 6.947342256583176, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 1.2988052368164062, + "mean_abs_diff": 0.2101205289363861, + "cosine_similarity": 0.9978237152099609, + "kl_divergence": 0.0007265903462690505, + "sigma_level": 15.178418190254549, + "cpk": 4.793697958538687, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 1.0872032642364502, + "mean_abs_diff": 0.16018898785114288, + "cosine_similarity": 0.9980590343475342, + "kl_divergence": 0.013828814951459963, + "sigma_level": 19.748898118047645, + "cpk": 6.3193363726237655, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.9903631210327148, + "mean_abs_diff": 0.17371346056461334, + "cosine_similarity": 0.9984952807426453, + "kl_divergence": 0.0038416856287773597, + "sigma_level": 18.438460185295785, + "cpk": 5.879236001242714, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.7914443016052246, + "mean_abs_diff": 0.13904337584972382, + "cosine_similarity": 0.9994173645973206, + "kl_divergence": 0.0011267766165267705, + "sigma_level": 23.019298841074793, + "cpk": 7.406376195311875, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 624, + "gpu_argmax": 624, + "max_abs_diff": 0.7683401107788086, + "mean_abs_diff": 0.11628174781799316, + "cosine_similarity": 0.9988746643066406, + "kl_divergence": 0.0021637816443614774, + "sigma_level": 27.418492188242922, + "cpk": 8.873808213232401, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.5380454063415527, + "mean_abs_diff": 0.10169024765491486, + "cosine_similarity": 0.999545693397522, + "kl_divergence": 0.004308393425260426, + "sigma_level": 31.753008418995396, + "cpk": 10.315255198838779, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 3974, + "gpu_argmax": 3974, + "max_abs_diff": 0.4585554599761963, + "mean_abs_diff": 0.06840988248586655, + "cosine_similarity": 0.9997379183769226, + "kl_divergence": 0.003921000934134561, + "sigma_level": 46.307180649881325, + "cpk": 15.17173781775126, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.5177440643310547, + "mean_abs_diff": 0.06941164284944534, + "cosine_similarity": 0.9996582865715027, + "kl_divergence": 0.0020656723212676966, + "sigma_level": 45.76503932690633, + "cpk": 14.99029422857294, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.420398473739624, + "mean_abs_diff": 0.06813345104455948, + "cosine_similarity": 0.9997531175613403, + "kl_divergence": 0.0012232872752691486, + "sigma_level": 46.26818246767472, + "cpk": 15.160026577134735, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.4519714117050171, + "mean_abs_diff": 0.0760267823934555, + "cosine_similarity": 0.9996703267097473, + "kl_divergence": 0.001647045695020044, + "sigma_level": 41.99132346083567, + "cpk": 13.731069052681043, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 9293, + "gpu_argmax": 9293, + "max_abs_diff": 0.44501709938049316, + "mean_abs_diff": 0.08212687820196152, + "cosine_similarity": 0.999630331993103, + "kl_divergence": 0.006501142529339967, + "sigma_level": 38.93182174829522, + "cpk": 12.710828167523179, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 369, + "gpu_argmax": 369, + "max_abs_diff": 0.46442174911499023, + "mean_abs_diff": 0.08806511759757996, + "cosine_similarity": 0.9996541738510132, + "kl_divergence": 0.0013393907643138331, + "sigma_level": 36.201662750395194, + "cpk": 11.801545609519941, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.41184115409851074, + "mean_abs_diff": 0.0735999047756195, + "cosine_similarity": 0.9997572898864746, + "kl_divergence": 0.0020352302058443735, + "sigma_level": 42.657347277985636, + "cpk": 13.957484367858523, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.39466023445129395, + "mean_abs_diff": 0.07817857712507248, + "cosine_similarity": 0.999573826789856, + "kl_divergence": 0.00016666892717610718, + "sigma_level": 41.49852594567503, + "cpk": 13.562484005956613, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.5057002902030945, + "mean_abs_diff": 0.07810698449611664, + "cosine_similarity": 0.9997146129608154, + "kl_divergence": 0.0018880510517696614, + "sigma_level": 40.865303812203365, + "cpk": 13.355779133127049, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 14310, + "gpu_argmax": 14310, + "max_abs_diff": 0.4656808376312256, + "mean_abs_diff": 0.07576054334640503, + "cosine_similarity": 0.9997501373291016, + "kl_divergence": 0.0019064405359464263, + "sigma_level": 42.13443236727582, + "cpk": 13.778800164947176, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.3974335193634033, + "mean_abs_diff": 0.07983855158090591, + "cosine_similarity": 0.999765157699585, + "kl_divergence": 0.003539420801547969, + "sigma_level": 40.48393264839967, + "cpk": 13.225296004054293, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.5480650663375854, + "mean_abs_diff": 0.09393318742513657, + "cosine_similarity": 0.999376118183136, + "kl_divergence": 0.0038084510052187473, + "sigma_level": 33.93247368937331, + "cpk": 11.045209112219235, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.5794413089752197, + "mean_abs_diff": 0.08977903425693512, + "cosine_similarity": 0.9995113015174866, + "kl_divergence": 0.002239943144359716, + "sigma_level": 34.99617146302287, + "cpk": 11.40356361462093, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.4805694818496704, + "mean_abs_diff": 0.07361047714948654, + "cosine_similarity": 0.9997532367706299, + "kl_divergence": 0.0017476027456557173, + "sigma_level": 43.46233730592578, + "cpk": 14.220838819715166, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.575049638748169, + "mean_abs_diff": 0.08266887068748474, + "cosine_similarity": 0.9996919631958008, + "kl_divergence": 0.005820226039888798, + "sigma_level": 38.466205868760724, + "cpk": 12.55707213968668, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4894747734069824, + "mean_abs_diff": 0.07466553151607513, + "cosine_similarity": 0.9998354911804199, + "kl_divergence": 0.001684058784819267, + "sigma_level": 42.247815077849495, + "cpk": 13.819733728601458, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.39302289485931396, + "mean_abs_diff": 0.0665874183177948, + "cosine_similarity": 0.9998313784599304, + "kl_divergence": 0.0034372790646476988, + "sigma_level": 47.353553428886435, + "cpk": 15.521755237045204, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.41148829460144043, + "mean_abs_diff": 0.0705053061246872, + "cosine_similarity": 0.9995219707489014, + "kl_divergence": 0.000481547916431422, + "sigma_level": 44.680539884015715, + "cpk": 14.63099536614367, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.4649146795272827, + "mean_abs_diff": 0.07782384008169174, + "cosine_similarity": 0.9995982646942139, + "kl_divergence": 0.0015583181127582582, + "sigma_level": 40.227398043234544, + "cpk": 13.148245131726577, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.47322654724121094, + "mean_abs_diff": 0.087704598903656, + "cosine_similarity": 0.9995580315589905, + "kl_divergence": 0.0022127799633092564, + "sigma_level": 36.708986647160415, + "cpk": 11.968033303216066, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.430513858795166, + "mean_abs_diff": 0.08133339881896973, + "cosine_similarity": 0.9997889995574951, + "kl_divergence": 0.00018704778171310847, + "sigma_level": 39.01572660971666, + "cpk": 12.740802065525553, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.45681631565093994, + "mean_abs_diff": 0.08105611801147461, + "cosine_similarity": 0.9997667074203491, + "kl_divergence": 0.003126264705790457, + "sigma_level": 39.84984885704753, + "cpk": 13.01411011470782, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.36823856830596924, + "mean_abs_diff": 0.061732012778520584, + "cosine_similarity": 0.9998225569725037, + "kl_divergence": 0.0028589001523262048, + "sigma_level": 50.88439404151175, + "cpk": 16.699698341904092, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 1075, + "gpu_argmax": 1075, + "max_abs_diff": 0.45197248458862305, + "mean_abs_diff": 0.06759438663721085, + "cosine_similarity": 0.9998044967651367, + "kl_divergence": 0.0044328146591125035, + "sigma_level": 46.359707613196264, + "cpk": 15.192097870999218, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.4539768099784851, + "mean_abs_diff": 0.07912351936101913, + "cosine_similarity": 0.9997790455818176, + "kl_divergence": 0.00013784763067440514, + "sigma_level": 40.56873603366942, + "cpk": 13.255416913638795, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.38960933685302734, + "mean_abs_diff": 0.06396492570638657, + "cosine_similarity": 0.9998103380203247, + "kl_divergence": 0.0029847960362235373, + "sigma_level": 49.1493031479779, + "cpk": 16.121115088960877, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.40729308128356934, + "mean_abs_diff": 0.06924924999475479, + "cosine_similarity": 0.9996633529663086, + "kl_divergence": 0.005449064420257557, + "sigma_level": 45.52429008065333, + "cpk": 14.912053114832036, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.36036384105682373, + "mean_abs_diff": 0.05879916250705719, + "cosine_similarity": 0.9997984170913696, + "kl_divergence": 0.00044588987994617586, + "sigma_level": 53.598492891250764, + "cpk": 17.60353542261309, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 8003, + "gpu_argmax": 17944, + "max_abs_diff": 0.4275398254394531, + "mean_abs_diff": 0.06924005597829819, + "cosine_similarity": 0.9997360110282898, + "kl_divergence": 0.0030625878432387578, + "sigma_level": 45.48962434668141, + "cpk": 14.900732770877472, + "verdict": "WarnArgmax" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 16293, + "gpu_argmax": 16293, + "max_abs_diff": 0.4567922353744507, + "mean_abs_diff": 0.07736830413341522, + "cosine_similarity": 0.9996188879013062, + "kl_divergence": 0.00627417830634242, + "sigma_level": 40.883418450180905, + "cpk": 13.36421608733803, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.42419886589050293, + "mean_abs_diff": 0.07781759649515152, + "cosine_similarity": 0.9997288584709167, + "kl_divergence": 0.00445967416527253, + "sigma_level": 40.85204776562423, + "cpk": 13.352431907789242, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2849, + "gpu_argmax": 2849, + "max_abs_diff": 0.41576290130615234, + "mean_abs_diff": 0.06628850847482681, + "cosine_similarity": 0.9997788071632385, + "kl_divergence": 0.002364125347954264, + "sigma_level": 47.590142022032, + "cpk": 15.600490712948526, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 15409, + "gpu_argmax": 15409, + "max_abs_diff": 0.4992527961730957, + "mean_abs_diff": 0.08212177455425262, + "cosine_similarity": 0.9995926022529602, + "kl_divergence": 0.0024074922898179433, + "sigma_level": 39.21766934638157, + "cpk": 12.804171065409959, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.5810892581939697, + "mean_abs_diff": 0.0682300254702568, + "cosine_similarity": 0.999836266040802, + "kl_divergence": 0.002712155435780897, + "sigma_level": 46.05452141964385, + "cpk": 15.08964870909105, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.43909335136413574, + "mean_abs_diff": 0.07037733495235443, + "cosine_similarity": 0.9998226165771484, + "kl_divergence": 0.0024886335454403913, + "sigma_level": 44.62151623300525, + "cpk": 14.612143461500741, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.5455688238143921, + "mean_abs_diff": 0.08337622880935669, + "cosine_similarity": 0.9997198581695557, + "kl_divergence": 0.004219288243595902, + "sigma_level": 38.8716398899516, + "cpk": 12.68713240151224, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.4702770709991455, + "mean_abs_diff": 0.07033935934305191, + "cosine_similarity": 0.9997967481613159, + "kl_divergence": 0.002269168837427208, + "sigma_level": 44.76284014389586, + "cpk": 14.658564256457195, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.47287213802337646, + "mean_abs_diff": 0.08464169502258301, + "cosine_similarity": 0.9996572136878967, + "kl_divergence": 0.0032647584277852933, + "sigma_level": 38.09939719641655, + "cpk": 12.431065935635237, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.403623104095459, + "mean_abs_diff": 0.08138824999332428, + "cosine_similarity": 0.9997397065162659, + "kl_divergence": 0.002429769070231016, + "sigma_level": 39.845518962009926, + "cpk": 13.011593232470492, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.41884738206863403, + "mean_abs_diff": 0.07357224076986313, + "cosine_similarity": 0.9997681975364685, + "kl_divergence": 0.0026198766285110034, + "sigma_level": 43.19010015433112, + "cpk": 14.131900680824627, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.38974857330322266, + "mean_abs_diff": 0.06174059584736824, + "cosine_similarity": 0.9997618794441223, + "kl_divergence": 0.0024988794185040867, + "sigma_level": 51.257036123035775, + "cpk": 16.8219587117114, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.4303196668624878, + "mean_abs_diff": 0.07956542819738388, + "cosine_similarity": 0.9997139573097229, + "kl_divergence": 0.000753836734309782, + "sigma_level": 40.610606016595504, + "cpk": 13.267601984109694, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.39939403533935547, + "mean_abs_diff": 0.06406113505363464, + "cosine_similarity": 0.9996994137763977, + "kl_divergence": 0.0021454389459888124, + "sigma_level": 49.09465897252818, + "cpk": 16.102798025938455, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4145479202270508, + "mean_abs_diff": 0.06807403266429901, + "cosine_similarity": 0.9997435808181763, + "kl_divergence": 0.0013922520524832468, + "sigma_level": 46.44678116489198, + "cpk": 15.218775413449793, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.39621949195861816, + "mean_abs_diff": 0.06436076015233994, + "cosine_similarity": 0.9998461008071899, + "kl_divergence": 0.0007129833951081995, + "sigma_level": 48.813617709656256, + "cpk": 16.009399108087152, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.5128415822982788, + "mean_abs_diff": 0.07186087220907211, + "cosine_similarity": 0.9997901320457458, + "kl_divergence": 0.0010620789511583434, + "sigma_level": 44.16569226501575, + "cpk": 14.457415324348458, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.4191385507583618, + "mean_abs_diff": 0.0679415762424469, + "cosine_similarity": 0.9998086094856262, + "kl_divergence": 0.0027788771123882566, + "sigma_level": 46.44338193961324, + "cpk": 15.218174265287137, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.36243438720703125, + "mean_abs_diff": 0.06031492352485657, + "cosine_similarity": 0.9998459815979004, + "kl_divergence": 0.0011979330012718125, + "sigma_level": 52.57512210986416, + "cpk": 17.26078533084085, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.4538910984992981, + "mean_abs_diff": 0.06918986886739731, + "cosine_similarity": 0.9997843503952026, + "kl_divergence": 0.0007243342259276751, + "sigma_level": 45.90852515083764, + "cpk": 15.038141314022376, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.44234466552734375, + "mean_abs_diff": 0.0634784922003746, + "cosine_similarity": 0.9998288154602051, + "kl_divergence": 0.0007674909766671851, + "sigma_level": 49.43904155443516, + "cpk": 16.21815420033612, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.351959228515625, + "mean_abs_diff": 0.05642838776111603, + "cosine_similarity": 0.999832808971405, + "kl_divergence": 0.0019997153002613145, + "sigma_level": 55.90082537578916, + "cpk": 18.370742337723765, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.38826751708984375, + "mean_abs_diff": 0.0642109364271164, + "cosine_similarity": 0.9998274445533752, + "kl_divergence": 0.0006867424058280134, + "sigma_level": 49.15437011428163, + "cpk": 16.121769360216955, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.37309741973876953, + "mean_abs_diff": 0.06227777898311615, + "cosine_similarity": 0.9998220205307007, + "kl_divergence": 0.0021385287649098024, + "sigma_level": 50.534553708367916, + "cpk": 16.58258625555096, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 18589, + "gpu_argmax": 18589, + "max_abs_diff": 0.3214547634124756, + "mean_abs_diff": 0.053083546459674835, + "cosine_similarity": 0.9998363256454468, + "kl_divergence": 0.000752778125991802, + "sigma_level": 59.39349988162695, + "cpk": 19.535098493011564, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.36084842681884766, + "mean_abs_diff": 0.061356306076049805, + "cosine_similarity": 0.9998142123222351, + "kl_divergence": 0.003312728169157689, + "sigma_level": 51.05759445120621, + "cpk": 16.75813936768082, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 7354, + "gpu_argmax": 7354, + "max_abs_diff": 0.44296157360076904, + "mean_abs_diff": 0.06914495676755905, + "cosine_similarity": 0.9997162818908691, + "kl_divergence": 0.0032034377972692007, + "sigma_level": 46.07774351870567, + "cpk": 15.093744207606258, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.4447762966156006, + "mean_abs_diff": 0.07214333862066269, + "cosine_similarity": 0.999789834022522, + "kl_divergence": 0.002128935912980073, + "sigma_level": 44.064428985081875, + "cpk": 14.42323007657755, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1033, + "gpu_argmax": 1033, + "max_abs_diff": 0.43563520908355713, + "mean_abs_diff": 0.07464952021837234, + "cosine_similarity": 0.9997342228889465, + "kl_divergence": 0.0023521526351508127, + "sigma_level": 42.89405481276757, + "cpk": 14.031183219923053, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.41919422149658203, + "mean_abs_diff": 0.07205758988857269, + "cosine_similarity": 0.9996944665908813, + "kl_divergence": 0.0015906886315562518, + "sigma_level": 44.11487020371673, + "cpk": 14.44005579914499, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.4340832233428955, + "mean_abs_diff": 0.06832505017518997, + "cosine_similarity": 0.9997145533561707, + "kl_divergence": 4.531155001990606e-05, + "sigma_level": 46.65343302362604, + "cpk": 15.285511161860004, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.5156793594360352, + "mean_abs_diff": 0.07876264303922653, + "cosine_similarity": 0.9997888803482056, + "kl_divergence": 0.002400534868525818, + "sigma_level": 40.82280172433786, + "cpk": 13.339657928106357, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.49131178855895996, + "mean_abs_diff": 0.06945641338825226, + "cosine_similarity": 0.9997475743293762, + "kl_divergence": 0.0013757893037735654, + "sigma_level": 45.35377881058699, + "cpk": 14.855417036046704, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 2390, + "gpu_argmax": 2390, + "max_abs_diff": 0.5032455325126648, + "mean_abs_diff": 0.07041440159082413, + "cosine_similarity": 0.9997410178184509, + "kl_divergence": 0.0025526453726194957, + "sigma_level": 44.967915760286665, + "cpk": 14.72543951350829, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.47355103492736816, + "mean_abs_diff": 0.08038556575775146, + "cosine_similarity": 0.999701976776123, + "kl_divergence": 0.0016015019282984591, + "sigma_level": 39.61497617254028, + "cpk": 12.939619368170968, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.4618046283721924, + "mean_abs_diff": 0.07349393516778946, + "cosine_similarity": 0.999594509601593, + "kl_divergence": 0.0007789350432452571, + "sigma_level": 43.43102772649833, + "cpk": 14.211016147499297, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 1865, + "max_abs_diff": 0.40026283264160156, + "mean_abs_diff": 0.06240736320614815, + "cosine_similarity": 0.9997082948684692, + "kl_divergence": 0.002672701304671866, + "sigma_level": 50.514450324264075, + "cpk": 16.57544397070925, + "verdict": "WarnArgmax" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.38044023513793945, + "mean_abs_diff": 0.06645609438419342, + "cosine_similarity": 0.9998005628585815, + "kl_divergence": 0.000939163318030381, + "sigma_level": 47.37136548013599, + "cpk": 15.528112165423995, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.4195518493652344, + "mean_abs_diff": 0.06238096207380295, + "cosine_similarity": 0.9997071623802185, + "kl_divergence": 0.0019081753538363014, + "sigma_level": 50.68498889665763, + "cpk": 16.631514768046415, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json index 386cde6b83..10d70e594d 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "c642576eecb62daa", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md heading '2026-09-06T13:5xZ' \u2014 the record states the hour and deliberately not the minute, so only the date is carried here", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9508274793624878, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "FAIL", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 15, - "gpu_argmax": 16, - "max_abs_diff": 11.973122596740723, - "mean_abs_diff": 1.4562522172927856, - "cosine_similarity": 0.9508274793624878, - "kl_divergence": 5.417725371363736, - "sigma_level": 2.14457759095467, - "cpk": 0.4546053743195434, - "verdict": "WarnOutOfSpec" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.4650428295135498, - "mean_abs_diff": 0.07376201450824738, - "cosine_similarity": 0.9998178482055664, - "kl_divergence": 0.000053598414848508557, - "sigma_level": 42.80674637067787, - "cpk": 14.005789469655557, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.3866511583328247, - "mean_abs_diff": 0.06590328365564346, - "cosine_similarity": 0.9998971819877625, - "kl_divergence": 6.680759951875907e-6, - "sigma_level": 47.80867514885475, - "cpk": 15.67366265965696, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.3977069854736328, - "mean_abs_diff": 0.05706659331917763, - "cosine_similarity": 0.9998915195465088, - "kl_divergence": 0.0000789326202787771, - "sigma_level": 54.87129607627002, - "cpk": 18.029488863916615, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.34780454635620117, - "mean_abs_diff": 0.05530937761068344, - "cosine_similarity": 0.9998860359191895, - "kl_divergence": 7.848848746470546e-6, - "sigma_level": 56.51914696416572, - "cpk": 18.579212417915677, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3383975028991699, - "mean_abs_diff": 0.05794849619269371, - "cosine_similarity": 0.9998785853385925, - "kl_divergence": 0.00012155244184822574, - "sigma_level": 55.224400447456055, - "cpk": 18.14145256922925, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.3957533836364746, - "mean_abs_diff": 0.06864283233880997, - "cosine_similarity": 0.9999083876609802, - "kl_divergence": 6.281946350910951e-6, - "sigma_level": 46.65570749970518, - "cpk": 15.28502084093916, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.4882650375366211, - "mean_abs_diff": 0.0651940405368805, - "cosine_similarity": 0.9999063014984131, - "kl_divergence": 0.000016794876271664552, - "sigma_level": 47.99269977987777, - "cpk": 15.73683009204895, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2981089949607849, - "mean_abs_diff": 0.05331605300307274, - "cosine_similarity": 0.9998162388801575, - "kl_divergence": 0.0004707982680592144, - "sigma_level": 59.65800432330056, - "cpk": 19.620940664386968, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.3762543201446533, - "mean_abs_diff": 0.056634485721588135, - "cosine_similarity": 0.9998602271080017, - "kl_divergence": 0.00200288227214394, - "sigma_level": 55.89523235415072, - "cpk": 18.367944306494742, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.3492332696914673, - "mean_abs_diff": 0.05141938105225563, - "cosine_similarity": 0.9999210238456726, - "kl_divergence": 0.002701770685362771, - "sigma_level": 60.75193355286236, - "cpk": 19.99032561586946, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.39516687393188477, - "mean_abs_diff": 0.06365680694580078, - "cosine_similarity": 0.9998645186424255, - "kl_divergence": 0.0017848021580618698, - "sigma_level": 50.93329113486925, - "cpk": 16.70757615488253, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.34313130378723145, - "mean_abs_diff": 0.04843280836939812, - "cosine_similarity": 0.999874472618103, - "kl_divergence": 0.0004959427385190338, - "sigma_level": 64.8719807942433, - "cpk": 21.362165913551863, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.33470678329467773, - "mean_abs_diff": 0.052488118410110474, - "cosine_similarity": 0.9998583793640137, - "kl_divergence": 0.0016068498758497576, - "sigma_level": 59.61238325877218, - "cpk": 19.610049266991116, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.3048872947692871, - "mean_abs_diff": 0.049873605370521545, - "cosine_similarity": 0.9998965859413147, - "kl_divergence": 0.0029217986943148653, - "sigma_level": 62.751582374651335, - "cpk": 20.656390153573017, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.4006218910217285, - "mean_abs_diff": 0.05147523805499077, - "cosine_similarity": 0.9998210072517395, - "kl_divergence": 0.0006769849863572717, - "sigma_level": 61.15841057435974, - "cpk": 20.123791546171578, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3487355709075928, - "mean_abs_diff": 0.05283060669898987, - "cosine_similarity": 0.9998836517333984, - "kl_divergence": 0.0022685891938408463, - "sigma_level": 59.646889061218864, - "cpk": 19.61969790900533, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.3330197334289551, - "mean_abs_diff": 0.052766453474760056, - "cosine_similarity": 0.9998737573623657, - "kl_divergence": 0.00006840857161954282, - "sigma_level": 59.305164302653225, - "cpk": 19.50761116813533, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.316272497177124, - "mean_abs_diff": 0.051003385335206985, - "cosine_similarity": 0.9998650550842285, - "kl_divergence": 0.000382817233105055, - "sigma_level": 61.19029400286672, - "cpk": 20.13668865563867, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3347015380859375, - "mean_abs_diff": 0.05140787735581398, - "cosine_similarity": 0.9998480081558228, - "kl_divergence": 0.0014383470256046078, - "sigma_level": 61.62469731210393, - "cpk": 20.277566197242162, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.3056960105895996, - "mean_abs_diff": 0.04827465862035751, - "cosine_similarity": 0.9998608231544495, - "kl_divergence": 0.0018885915278280133, - "sigma_level": 64.61486353567884, - "cpk": 21.278349471977457, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.6572532653808594, - "mean_abs_diff": 0.10043209791183472, - "cosine_similarity": 0.9996307492256165, - "kl_divergence": 0.005978000238223978, - "sigma_level": 31.087704271296346, - "cpk": 10.102384477163033, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.5020737648010254, - "mean_abs_diff": 0.07753492891788483, - "cosine_similarity": 0.9997056126594543, - "kl_divergence": 0.0061177710560870804, - "sigma_level": 40.33543045360021, - "cpk": 13.184526423442389, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3872443437576294, - "mean_abs_diff": 0.062455497682094574, - "cosine_similarity": 0.9998182654380798, - "kl_divergence": 0.0008687540237392052, - "sigma_level": 50.5086856784924, - "cpk": 16.57334980105424, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.3850289583206177, - "mean_abs_diff": 0.06431932002305984, - "cosine_similarity": 0.9998119473457336, - "kl_divergence": 0.002553620620018977, - "sigma_level": 49.01901527706649, - "cpk": 16.076932614787086, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3047807812690735, - "mean_abs_diff": 0.04829900339245796, - "cosine_similarity": 0.999798059463501, - "kl_divergence": 0.00021319612052604753, - "sigma_level": 66.28606747309414, - "cpk": 21.828559907885072, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3059554100036621, - "mean_abs_diff": 0.048369839787483215, - "cosine_similarity": 0.9998838901519775, - "kl_divergence": 0.0006849703836857524, - "sigma_level": 65.17807355881911, - "cpk": 21.463303438298297, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.41999053955078125, - "mean_abs_diff": 0.07317500561475754, - "cosine_similarity": 0.9994683265686035, - "kl_divergence": 0.0009813839538431752, - "sigma_level": 43.85812919626951, - "cpk": 14.351933161240689, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.35340678691864014, - "mean_abs_diff": 0.056793734431266785, - "cosine_similarity": 0.999854326248169, - "kl_divergence": 0.0038247994839233756, - "sigma_level": 55.143344205449615, - "cpk": 18.120131697945187, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.35067176818847656, - "mean_abs_diff": 0.0629408061504364, - "cosine_similarity": 0.9997932314872742, - "kl_divergence": 0.0006541521126937118, - "sigma_level": 50.225147145014844, - "cpk": 16.478281444143988, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.29090166091918945, - "mean_abs_diff": 0.04321891814470291, - "cosine_similarity": 0.9998005628585815, - "kl_divergence": 0.00004718849190647301, - "sigma_level": 72.6417500127864, - "cpk": 23.952291850287917, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2871342897415161, - "mean_abs_diff": 0.044014591723680496, - "cosine_similarity": 0.9999145865440369, - "kl_divergence": 0.0004054254268827301, - "sigma_level": 71.05546670016032, - "cpk": 23.424532453674846, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.280792236328125, - "mean_abs_diff": 0.045980505645275116, - "cosine_similarity": 0.9999239444732666, - "kl_divergence": 0.001747060931358211, - "sigma_level": 68.52831234877785, - "cpk": 22.580190245191435, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 1075, - "max_abs_diff": 0.2834291458129883, - "mean_abs_diff": 0.04462064057588577, - "cosine_similarity": 0.9999250769615173, - "kl_divergence": 0.0019013842820190484, - "sigma_level": 70.39793470719678, - "cpk": 23.204211490577716, - "verdict": "WarnArgmax" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.31037187576293945, - "mean_abs_diff": 0.052936043590307236, - "cosine_similarity": 0.9998127222061157, - "kl_divergence": 0.00005010117795112397, - "sigma_level": 61.08032036537772, - "cpk": 20.090660913344944, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.35306501388549805, - "mean_abs_diff": 0.05245150998234749, - "cosine_similarity": 0.9999088644981384, - "kl_divergence": 0.0014205977063074006, - "sigma_level": 59.50700084759674, - "cpk": 19.575564278450805, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.4437136650085449, - "mean_abs_diff": 0.0651242807507515, - "cosine_similarity": 0.999549925327301, - "kl_divergence": 0.002034610592857552, - "sigma_level": 47.985960776301, - "cpk": 15.734899326959468, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.2869229316711426, - "mean_abs_diff": 0.04254760593175888, - "cosine_similarity": 0.9998387098312378, - "kl_divergence": 0.00012317854697163394, - "sigma_level": 73.4563462456492, - "cpk": 24.22499944244584, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.31911468505859375, - "mean_abs_diff": 0.04899978265166283, - "cosine_similarity": 0.9999270439147949, - "kl_divergence": 0.001279600249078306, - "sigma_level": 64.5414262336222, - "cpk": 21.25026575641775, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3646974563598633, - "mean_abs_diff": 0.07297120988368988, - "cosine_similarity": 0.9998273253440857, - "kl_divergence": 0.001634865324591819, - "sigma_level": 45.15382628552608, - "cpk": 14.77669798393096, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.3667411804199219, - "mean_abs_diff": 0.07524772733449936, - "cosine_similarity": 0.9998779892921448, - "kl_divergence": 0.001175862369518911, - "sigma_level": 44.43649313589075, - "cpk": 14.533518952031011, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.28829050064086914, - "mean_abs_diff": 0.0564601756632328, - "cosine_similarity": 0.9999026656150818, - "kl_divergence": 0.0012793447849670605, - "sigma_level": 57.344280001367196, - "cpk": 18.844954323609166, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.2895240783691406, - "mean_abs_diff": 0.045262835919857025, - "cosine_similarity": 0.999929666519165, - "kl_divergence": 0.0014526099267751793, - "sigma_level": 69.36425130160303, - "cpk": 22.859781873420328, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.30231142044067383, - "mean_abs_diff": 0.04469147324562073, - "cosine_similarity": 0.9998422265052795, - "kl_divergence": 0.0002661089801215439, - "sigma_level": 71.27346144399971, - "cpk": 23.492377481729296, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.31220388412475586, - "mean_abs_diff": 0.0626152977347374, - "cosine_similarity": 0.9998358488082886, - "kl_divergence": 0.0012580873445873658, - "sigma_level": 52.37533010732988, - "cpk": 17.185151961724493, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.3172950744628906, - "mean_abs_diff": 0.056548990309238434, - "cosine_similarity": 0.9998992085456848, - "kl_divergence": 0.0019092303061413402, - "sigma_level": 57.02370802279627, - "cpk": 18.73918324823393, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.287722110748291, - "mean_abs_diff": 0.046183329075574875, - "cosine_similarity": 0.9998865127563477, - "kl_divergence": 0.001074622186835361, - "sigma_level": 67.91322827569736, - "cpk": 22.376371177729023, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.28690052032470703, - "mean_abs_diff": 0.04460417479276657, - "cosine_similarity": 0.9998990297317505, - "kl_divergence": 0.001637326459726193, - "sigma_level": 70.32867329867013, - "cpk": 23.181478396493603, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.2437753677368164, - "mean_abs_diff": 0.042050521820783615, - "cosine_similarity": 0.9999042749404907, - "kl_divergence": 0.0012921399977282274, - "sigma_level": 75.96778021835588, - "cpk": 25.056386339472922, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.26329851150512695, - "mean_abs_diff": 0.04378020763397217, - "cosine_similarity": 0.9998823404312134, - "kl_divergence": 0.00045292527632992516, - "sigma_level": 72.18207270711018, - "cpk": 23.79734539148941, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.3285568952560425, - "mean_abs_diff": 0.04826463386416435, - "cosine_similarity": 0.9999091625213623, - "kl_divergence": 0.003169107297679868, - "sigma_level": 64.72904795942044, - "cpk": 21.316005669795366, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 1030, - "gpu_argmax": 1030, - "max_abs_diff": 0.2997932434082031, - "mean_abs_diff": 0.05100039765238762, - "cosine_similarity": 0.9998959302902222, - "kl_divergence": 0.0007471735698977093, - "sigma_level": 62.57656831041338, - "cpk": 20.592903614508387, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.35276174545288086, - "mean_abs_diff": 0.054895516484975815, - "cosine_similarity": 0.9999022483825684, - "kl_divergence": 0.0010573862584377517, - "sigma_level": 58.83785079955426, - "cpk": 19.343455749142464, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.23547744750976562, - "mean_abs_diff": 0.03824745863676071, - "cosine_similarity": 0.9999030828475952, - "kl_divergence": 0.00013315070766980216, - "sigma_level": 82.05800083254637, - "cpk": 27.091124444793948, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.23178386688232422, - "mean_abs_diff": 0.0405409075319767, - "cosine_similarity": 0.9998529553413391, - "kl_divergence": 0.00007050667318571868, - "sigma_level": 78.69024316368039, - "cpk": 25.96423323191284, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.2403573989868164, - "mean_abs_diff": 0.04604782164096832, - "cosine_similarity": 0.9998740553855896, - "kl_divergence": 0.0012601311136737736, - "sigma_level": 69.5124300500558, - "cpk": 22.904068684953998, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.2761220932006836, - "mean_abs_diff": 0.046715147793293, - "cosine_similarity": 0.9999170303344727, - "kl_divergence": 0.0011468693795866846, - "sigma_level": 68.6136821228989, - "cpk": 22.604119182548526, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.32673943042755127, - "mean_abs_diff": 0.042964544147253036, - "cosine_similarity": 0.999886691570282, - "kl_divergence": 0.0004910417376588438, - "sigma_level": 72.59800358871821, - "cpk": 23.939406185390244, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.2835589647293091, - "mean_abs_diff": 0.042426157742738724, - "cosine_similarity": 0.9998935461044312, - "kl_divergence": 0.00015988472627614193, - "sigma_level": 73.71183167144353, - "cpk": 24.310001407314605, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.26973533630371094, - "mean_abs_diff": 0.04785580188035965, - "cosine_similarity": 0.9999091625213623, - "kl_divergence": 0.00010123936424381408, - "sigma_level": 67.38781163452288, - "cpk": 22.193862397946567, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.28846168518066406, - "mean_abs_diff": 0.04364936426281929, - "cosine_similarity": 0.9999343156814575, - "kl_divergence": 0.0013821431895069671, - "sigma_level": 72.21723594234979, - "cpk": 23.809725610974798, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.26705145835876465, - "mean_abs_diff": 0.045417461544275284, - "cosine_similarity": 0.9998982548713684, - "kl_divergence": 0.00017813759490774671, - "sigma_level": 70.84042781016798, - "cpk": 23.34535990290197, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.3231534957885742, - "mean_abs_diff": 0.06047097593545914, - "cosine_similarity": 0.9998999834060669, - "kl_divergence": 0.0016757824665770912, - "sigma_level": 53.974261257017965, - "cpk": 17.719430731205378, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2636291980743408, - "mean_abs_diff": 0.04413612186908722, - "cosine_similarity": 0.9998951554298401, - "kl_divergence": 0.00045192088792373277, - "sigma_level": 71.99334230002293, - "cpk": 23.732988522547917, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2389669418334961, - "mean_abs_diff": 0.04086123779416084, - "cosine_similarity": 0.9999276995658875, - "kl_divergence": 0.0008661659413689716, - "sigma_level": 76.81578693588703, - "cpk": 25.343696633934606, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.25644922256469727, - "mean_abs_diff": 0.04367988184094429, - "cosine_similarity": 0.9998958706855774, - "kl_divergence": 0.001030973317421363, - "sigma_level": 71.68527909970481, - "cpk": 23.63415932316742, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.26444530487060547, - "mean_abs_diff": 0.05307325720787048, - "cosine_similarity": 0.9998828172683716, - "kl_divergence": 0.00022194838380929847, - "sigma_level": 61.76488152384485, - "cpk": 20.31512188765423, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.3115396499633789, - "mean_abs_diff": 0.054525721818208694, - "cosine_similarity": 0.9999011754989624, - "kl_divergence": 0.0005107738590351378, - "sigma_level": 59.64675652494656, - "cpk": 19.61122863717905, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.2833176851272583, - "mean_abs_diff": 0.0467853918671608, - "cosine_similarity": 0.9998739361763, - "kl_divergence": 0.0015430497955592104, - "sigma_level": 67.90594397032366, - "cpk": 22.370564140211133, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2992556095123291, - "mean_abs_diff": 0.05715445801615715, - "cosine_similarity": 0.9998778104782104, - "kl_divergence": 4.221885622172819e-6, - "sigma_level": 57.44807291346208, - "cpk": 18.875739848533893, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.24626314640045166, - "mean_abs_diff": 0.04180094972252846, - "cosine_similarity": 0.9998927712440491, - "kl_divergence": 0.00018937640439664716, - "sigma_level": 74.82955276570739, - "cpk": 24.68252205749258, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3256549835205078, - "mean_abs_diff": 0.05863361805677414, - "cosine_similarity": 0.9999057650566101, - "kl_divergence": 0.0011160061469011708, - "sigma_level": 55.66880028635367, - "cpk": 18.284261497645485, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.40552783012390137, - "mean_abs_diff": 0.06164288520812988, - "cosine_similarity": 0.9998986721038818, - "kl_divergence": 0.0016408893623984504, - "sigma_level": 52.06927285767559, - "cpk": 17.088949268422155, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.30518674850463867, - "mean_abs_diff": 0.052556075155735016, - "cosine_similarity": 0.9998877048492432, - "kl_divergence": 0.0011176506364210586, - "sigma_level": 60.6323717384759, - "cpk": 19.945240622328814, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2461402416229248, - "mean_abs_diff": 0.04279375076293945, - "cosine_similarity": 0.9999181628227234, - "kl_divergence": 0.00028949609438141944, - "sigma_level": 73.38914463933241, - "cpk": 24.201331816077396, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.28456664085388184, - "mean_abs_diff": 0.044448915868997574, - "cosine_similarity": 0.9999171495437622, - "kl_divergence": 0.0006934676863507526, - "sigma_level": 70.69053702108894, - "cpk": 23.301669195964262, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2627677917480469, - "mean_abs_diff": 0.04326142370700836, - "cosine_similarity": 0.9999173283576965, - "kl_divergence": 0.0003628763594607102, - "sigma_level": 73.39988060423036, - "cpk": 24.2020115901715, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.28603506088256836, - "mean_abs_diff": 0.04886169731616974, - "cosine_similarity": 0.9998109340667725, - "kl_divergence": 0.0013024145853326003, - "sigma_level": 64.2417199046864, - "cpk": 21.15232667880772, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 15, + "gpu_argmax": 16, + "max_abs_diff": 11.973122596740723, + "mean_abs_diff": 1.4562522172927856, + "cosine_similarity": 0.9508274793624878, + "kl_divergence": 5.417725371363736, + "sigma_level": 2.14457759095467, + "cpk": 0.4546053743195434, + "verdict": "WarnOutOfSpec" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.4650428295135498, + "mean_abs_diff": 0.07376201450824738, + "cosine_similarity": 0.9998178482055664, + "kl_divergence": 5.3598414848508557e-05, + "sigma_level": 42.80674637067787, + "cpk": 14.005789469655557, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.3866511583328247, + "mean_abs_diff": 0.06590328365564346, + "cosine_similarity": 0.9998971819877625, + "kl_divergence": 6.680759951875907e-06, + "sigma_level": 47.80867514885475, + "cpk": 15.67366265965696, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.3977069854736328, + "mean_abs_diff": 0.05706659331917763, + "cosine_similarity": 0.9998915195465088, + "kl_divergence": 7.89326202787771e-05, + "sigma_level": 54.87129607627002, + "cpk": 18.029488863916615, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.34780454635620117, + "mean_abs_diff": 0.05530937761068344, + "cosine_similarity": 0.9998860359191895, + "kl_divergence": 7.848848746470546e-06, + "sigma_level": 56.51914696416572, + "cpk": 18.579212417915677, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3383975028991699, + "mean_abs_diff": 0.05794849619269371, + "cosine_similarity": 0.9998785853385925, + "kl_divergence": 0.00012155244184822574, + "sigma_level": 55.224400447456055, + "cpk": 18.14145256922925, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.3957533836364746, + "mean_abs_diff": 0.06864283233880997, + "cosine_similarity": 0.9999083876609802, + "kl_divergence": 6.281946350910951e-06, + "sigma_level": 46.65570749970518, + "cpk": 15.28502084093916, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.4882650375366211, + "mean_abs_diff": 0.0651940405368805, + "cosine_similarity": 0.9999063014984131, + "kl_divergence": 1.6794876271664552e-05, + "sigma_level": 47.99269977987777, + "cpk": 15.73683009204895, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2981089949607849, + "mean_abs_diff": 0.05331605300307274, + "cosine_similarity": 0.9998162388801575, + "kl_divergence": 0.0004707982680592144, + "sigma_level": 59.65800432330056, + "cpk": 19.620940664386968, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.3762543201446533, + "mean_abs_diff": 0.056634485721588135, + "cosine_similarity": 0.9998602271080017, + "kl_divergence": 0.00200288227214394, + "sigma_level": 55.89523235415072, + "cpk": 18.367944306494742, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.3492332696914673, + "mean_abs_diff": 0.05141938105225563, + "cosine_similarity": 0.9999210238456726, + "kl_divergence": 0.002701770685362771, + "sigma_level": 60.75193355286236, + "cpk": 19.99032561586946, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.39516687393188477, + "mean_abs_diff": 0.06365680694580078, + "cosine_similarity": 0.9998645186424255, + "kl_divergence": 0.0017848021580618698, + "sigma_level": 50.93329113486925, + "cpk": 16.70757615488253, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.34313130378723145, + "mean_abs_diff": 0.04843280836939812, + "cosine_similarity": 0.999874472618103, + "kl_divergence": 0.0004959427385190338, + "sigma_level": 64.8719807942433, + "cpk": 21.362165913551863, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.33470678329467773, + "mean_abs_diff": 0.052488118410110474, + "cosine_similarity": 0.9998583793640137, + "kl_divergence": 0.0016068498758497576, + "sigma_level": 59.61238325877218, + "cpk": 19.610049266991116, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.3048872947692871, + "mean_abs_diff": 0.049873605370521545, + "cosine_similarity": 0.9998965859413147, + "kl_divergence": 0.0029217986943148653, + "sigma_level": 62.751582374651335, + "cpk": 20.656390153573017, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.4006218910217285, + "mean_abs_diff": 0.05147523805499077, + "cosine_similarity": 0.9998210072517395, + "kl_divergence": 0.0006769849863572717, + "sigma_level": 61.15841057435974, + "cpk": 20.123791546171578, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3487355709075928, + "mean_abs_diff": 0.05283060669898987, + "cosine_similarity": 0.9998836517333984, + "kl_divergence": 0.0022685891938408463, + "sigma_level": 59.646889061218864, + "cpk": 19.61969790900533, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.3330197334289551, + "mean_abs_diff": 0.052766453474760056, + "cosine_similarity": 0.9998737573623657, + "kl_divergence": 6.840857161954282e-05, + "sigma_level": 59.305164302653225, + "cpk": 19.50761116813533, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.316272497177124, + "mean_abs_diff": 0.051003385335206985, + "cosine_similarity": 0.9998650550842285, + "kl_divergence": 0.000382817233105055, + "sigma_level": 61.19029400286672, + "cpk": 20.13668865563867, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3347015380859375, + "mean_abs_diff": 0.05140787735581398, + "cosine_similarity": 0.9998480081558228, + "kl_divergence": 0.0014383470256046078, + "sigma_level": 61.62469731210393, + "cpk": 20.277566197242162, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.3056960105895996, + "mean_abs_diff": 0.04827465862035751, + "cosine_similarity": 0.9998608231544495, + "kl_divergence": 0.0018885915278280133, + "sigma_level": 64.61486353567884, + "cpk": 21.278349471977457, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.6572532653808594, + "mean_abs_diff": 0.10043209791183472, + "cosine_similarity": 0.9996307492256165, + "kl_divergence": 0.005978000238223978, + "sigma_level": 31.087704271296346, + "cpk": 10.102384477163033, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.5020737648010254, + "mean_abs_diff": 0.07753492891788483, + "cosine_similarity": 0.9997056126594543, + "kl_divergence": 0.0061177710560870804, + "sigma_level": 40.33543045360021, + "cpk": 13.184526423442389, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3872443437576294, + "mean_abs_diff": 0.062455497682094574, + "cosine_similarity": 0.9998182654380798, + "kl_divergence": 0.0008687540237392052, + "sigma_level": 50.5086856784924, + "cpk": 16.57334980105424, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.3850289583206177, + "mean_abs_diff": 0.06431932002305984, + "cosine_similarity": 0.9998119473457336, + "kl_divergence": 0.002553620620018977, + "sigma_level": 49.01901527706649, + "cpk": 16.076932614787086, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3047807812690735, + "mean_abs_diff": 0.04829900339245796, + "cosine_similarity": 0.999798059463501, + "kl_divergence": 0.00021319612052604753, + "sigma_level": 66.28606747309414, + "cpk": 21.828559907885072, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3059554100036621, + "mean_abs_diff": 0.048369839787483215, + "cosine_similarity": 0.9998838901519775, + "kl_divergence": 0.0006849703836857524, + "sigma_level": 65.17807355881911, + "cpk": 21.463303438298297, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.41999053955078125, + "mean_abs_diff": 0.07317500561475754, + "cosine_similarity": 0.9994683265686035, + "kl_divergence": 0.0009813839538431752, + "sigma_level": 43.85812919626951, + "cpk": 14.351933161240689, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.35340678691864014, + "mean_abs_diff": 0.056793734431266785, + "cosine_similarity": 0.999854326248169, + "kl_divergence": 0.0038247994839233756, + "sigma_level": 55.143344205449615, + "cpk": 18.120131697945187, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.35067176818847656, + "mean_abs_diff": 0.0629408061504364, + "cosine_similarity": 0.9997932314872742, + "kl_divergence": 0.0006541521126937118, + "sigma_level": 50.225147145014844, + "cpk": 16.478281444143988, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.29090166091918945, + "mean_abs_diff": 0.04321891814470291, + "cosine_similarity": 0.9998005628585815, + "kl_divergence": 4.718849190647301e-05, + "sigma_level": 72.6417500127864, + "cpk": 23.952291850287917, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2871342897415161, + "mean_abs_diff": 0.044014591723680496, + "cosine_similarity": 0.9999145865440369, + "kl_divergence": 0.0004054254268827301, + "sigma_level": 71.05546670016032, + "cpk": 23.424532453674846, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.280792236328125, + "mean_abs_diff": 0.045980505645275116, + "cosine_similarity": 0.9999239444732666, + "kl_divergence": 0.001747060931358211, + "sigma_level": 68.52831234877785, + "cpk": 22.580190245191435, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 1075, + "max_abs_diff": 0.2834291458129883, + "mean_abs_diff": 0.04462064057588577, + "cosine_similarity": 0.9999250769615173, + "kl_divergence": 0.0019013842820190484, + "sigma_level": 70.39793470719678, + "cpk": 23.204211490577716, + "verdict": "WarnArgmax" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.31037187576293945, + "mean_abs_diff": 0.052936043590307236, + "cosine_similarity": 0.9998127222061157, + "kl_divergence": 5.010117795112397e-05, + "sigma_level": 61.08032036537772, + "cpk": 20.090660913344944, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.35306501388549805, + "mean_abs_diff": 0.05245150998234749, + "cosine_similarity": 0.9999088644981384, + "kl_divergence": 0.0014205977063074006, + "sigma_level": 59.50700084759674, + "cpk": 19.575564278450805, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.4437136650085449, + "mean_abs_diff": 0.0651242807507515, + "cosine_similarity": 0.999549925327301, + "kl_divergence": 0.002034610592857552, + "sigma_level": 47.985960776301, + "cpk": 15.734899326959468, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.2869229316711426, + "mean_abs_diff": 0.04254760593175888, + "cosine_similarity": 0.9998387098312378, + "kl_divergence": 0.00012317854697163394, + "sigma_level": 73.4563462456492, + "cpk": 24.22499944244584, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.31911468505859375, + "mean_abs_diff": 0.04899978265166283, + "cosine_similarity": 0.9999270439147949, + "kl_divergence": 0.001279600249078306, + "sigma_level": 64.5414262336222, + "cpk": 21.25026575641775, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3646974563598633, + "mean_abs_diff": 0.07297120988368988, + "cosine_similarity": 0.9998273253440857, + "kl_divergence": 0.001634865324591819, + "sigma_level": 45.15382628552608, + "cpk": 14.77669798393096, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.3667411804199219, + "mean_abs_diff": 0.07524772733449936, + "cosine_similarity": 0.9998779892921448, + "kl_divergence": 0.001175862369518911, + "sigma_level": 44.43649313589075, + "cpk": 14.533518952031011, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.28829050064086914, + "mean_abs_diff": 0.0564601756632328, + "cosine_similarity": 0.9999026656150818, + "kl_divergence": 0.0012793447849670605, + "sigma_level": 57.344280001367196, + "cpk": 18.844954323609166, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.2895240783691406, + "mean_abs_diff": 0.045262835919857025, + "cosine_similarity": 0.999929666519165, + "kl_divergence": 0.0014526099267751793, + "sigma_level": 69.36425130160303, + "cpk": 22.859781873420328, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.30231142044067383, + "mean_abs_diff": 0.04469147324562073, + "cosine_similarity": 0.9998422265052795, + "kl_divergence": 0.0002661089801215439, + "sigma_level": 71.27346144399971, + "cpk": 23.492377481729296, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.31220388412475586, + "mean_abs_diff": 0.0626152977347374, + "cosine_similarity": 0.9998358488082886, + "kl_divergence": 0.0012580873445873658, + "sigma_level": 52.37533010732988, + "cpk": 17.185151961724493, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.3172950744628906, + "mean_abs_diff": 0.056548990309238434, + "cosine_similarity": 0.9998992085456848, + "kl_divergence": 0.0019092303061413402, + "sigma_level": 57.02370802279627, + "cpk": 18.73918324823393, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.287722110748291, + "mean_abs_diff": 0.046183329075574875, + "cosine_similarity": 0.9998865127563477, + "kl_divergence": 0.001074622186835361, + "sigma_level": 67.91322827569736, + "cpk": 22.376371177729023, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.28690052032470703, + "mean_abs_diff": 0.04460417479276657, + "cosine_similarity": 0.9998990297317505, + "kl_divergence": 0.001637326459726193, + "sigma_level": 70.32867329867013, + "cpk": 23.181478396493603, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.2437753677368164, + "mean_abs_diff": 0.042050521820783615, + "cosine_similarity": 0.9999042749404907, + "kl_divergence": 0.0012921399977282274, + "sigma_level": 75.96778021835588, + "cpk": 25.056386339472922, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.26329851150512695, + "mean_abs_diff": 0.04378020763397217, + "cosine_similarity": 0.9998823404312134, + "kl_divergence": 0.00045292527632992516, + "sigma_level": 72.18207270711018, + "cpk": 23.79734539148941, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.3285568952560425, + "mean_abs_diff": 0.04826463386416435, + "cosine_similarity": 0.9999091625213623, + "kl_divergence": 0.003169107297679868, + "sigma_level": 64.72904795942044, + "cpk": 21.316005669795366, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 1030, + "gpu_argmax": 1030, + "max_abs_diff": 0.2997932434082031, + "mean_abs_diff": 0.05100039765238762, + "cosine_similarity": 0.9998959302902222, + "kl_divergence": 0.0007471735698977093, + "sigma_level": 62.57656831041338, + "cpk": 20.592903614508387, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.35276174545288086, + "mean_abs_diff": 0.054895516484975815, + "cosine_similarity": 0.9999022483825684, + "kl_divergence": 0.0010573862584377517, + "sigma_level": 58.83785079955426, + "cpk": 19.343455749142464, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.23547744750976562, + "mean_abs_diff": 0.03824745863676071, + "cosine_similarity": 0.9999030828475952, + "kl_divergence": 0.00013315070766980216, + "sigma_level": 82.05800083254637, + "cpk": 27.091124444793948, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.23178386688232422, + "mean_abs_diff": 0.0405409075319767, + "cosine_similarity": 0.9998529553413391, + "kl_divergence": 7.050667318571868e-05, + "sigma_level": 78.69024316368039, + "cpk": 25.96423323191284, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.2403573989868164, + "mean_abs_diff": 0.04604782164096832, + "cosine_similarity": 0.9998740553855896, + "kl_divergence": 0.0012601311136737736, + "sigma_level": 69.5124300500558, + "cpk": 22.904068684953998, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.2761220932006836, + "mean_abs_diff": 0.046715147793293, + "cosine_similarity": 0.9999170303344727, + "kl_divergence": 0.0011468693795866846, + "sigma_level": 68.6136821228989, + "cpk": 22.604119182548526, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.32673943042755127, + "mean_abs_diff": 0.042964544147253036, + "cosine_similarity": 0.999886691570282, + "kl_divergence": 0.0004910417376588438, + "sigma_level": 72.59800358871821, + "cpk": 23.939406185390244, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.2835589647293091, + "mean_abs_diff": 0.042426157742738724, + "cosine_similarity": 0.9998935461044312, + "kl_divergence": 0.00015988472627614193, + "sigma_level": 73.71183167144353, + "cpk": 24.310001407314605, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.26973533630371094, + "mean_abs_diff": 0.04785580188035965, + "cosine_similarity": 0.9999091625213623, + "kl_divergence": 0.00010123936424381408, + "sigma_level": 67.38781163452288, + "cpk": 22.193862397946567, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.28846168518066406, + "mean_abs_diff": 0.04364936426281929, + "cosine_similarity": 0.9999343156814575, + "kl_divergence": 0.0013821431895069671, + "sigma_level": 72.21723594234979, + "cpk": 23.809725610974798, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.26705145835876465, + "mean_abs_diff": 0.045417461544275284, + "cosine_similarity": 0.9998982548713684, + "kl_divergence": 0.00017813759490774671, + "sigma_level": 70.84042781016798, + "cpk": 23.34535990290197, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.3231534957885742, + "mean_abs_diff": 0.06047097593545914, + "cosine_similarity": 0.9998999834060669, + "kl_divergence": 0.0016757824665770912, + "sigma_level": 53.974261257017965, + "cpk": 17.719430731205378, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2636291980743408, + "mean_abs_diff": 0.04413612186908722, + "cosine_similarity": 0.9998951554298401, + "kl_divergence": 0.00045192088792373277, + "sigma_level": 71.99334230002293, + "cpk": 23.732988522547917, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2389669418334961, + "mean_abs_diff": 0.04086123779416084, + "cosine_similarity": 0.9999276995658875, + "kl_divergence": 0.0008661659413689716, + "sigma_level": 76.81578693588703, + "cpk": 25.343696633934606, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.25644922256469727, + "mean_abs_diff": 0.04367988184094429, + "cosine_similarity": 0.9998958706855774, + "kl_divergence": 0.001030973317421363, + "sigma_level": 71.68527909970481, + "cpk": 23.63415932316742, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.26444530487060547, + "mean_abs_diff": 0.05307325720787048, + "cosine_similarity": 0.9998828172683716, + "kl_divergence": 0.00022194838380929847, + "sigma_level": 61.76488152384485, + "cpk": 20.31512188765423, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.3115396499633789, + "mean_abs_diff": 0.054525721818208694, + "cosine_similarity": 0.9999011754989624, + "kl_divergence": 0.0005107738590351378, + "sigma_level": 59.64675652494656, + "cpk": 19.61122863717905, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.2833176851272583, + "mean_abs_diff": 0.0467853918671608, + "cosine_similarity": 0.9998739361763, + "kl_divergence": 0.0015430497955592104, + "sigma_level": 67.90594397032366, + "cpk": 22.370564140211133, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2992556095123291, + "mean_abs_diff": 0.05715445801615715, + "cosine_similarity": 0.9998778104782104, + "kl_divergence": 4.221885622172819e-06, + "sigma_level": 57.44807291346208, + "cpk": 18.875739848533893, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.24626314640045166, + "mean_abs_diff": 0.04180094972252846, + "cosine_similarity": 0.9998927712440491, + "kl_divergence": 0.00018937640439664716, + "sigma_level": 74.82955276570739, + "cpk": 24.68252205749258, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3256549835205078, + "mean_abs_diff": 0.05863361805677414, + "cosine_similarity": 0.9999057650566101, + "kl_divergence": 0.0011160061469011708, + "sigma_level": 55.66880028635367, + "cpk": 18.284261497645485, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.40552783012390137, + "mean_abs_diff": 0.06164288520812988, + "cosine_similarity": 0.9998986721038818, + "kl_divergence": 0.0016408893623984504, + "sigma_level": 52.06927285767559, + "cpk": 17.088949268422155, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.30518674850463867, + "mean_abs_diff": 0.052556075155735016, + "cosine_similarity": 0.9998877048492432, + "kl_divergence": 0.0011176506364210586, + "sigma_level": 60.6323717384759, + "cpk": 19.945240622328814, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2461402416229248, + "mean_abs_diff": 0.04279375076293945, + "cosine_similarity": 0.9999181628227234, + "kl_divergence": 0.00028949609438141944, + "sigma_level": 73.38914463933241, + "cpk": 24.201331816077396, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.28456664085388184, + "mean_abs_diff": 0.044448915868997574, + "cosine_similarity": 0.9999171495437622, + "kl_divergence": 0.0006934676863507526, + "sigma_level": 70.69053702108894, + "cpk": 23.301669195964262, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2627677917480469, + "mean_abs_diff": 0.04326142370700836, + "cosine_similarity": 0.9999173283576965, + "kl_divergence": 0.0003628763594607102, + "sigma_level": 73.39988060423036, + "cpk": 24.2020115901715, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.28603506088256836, + "mean_abs_diff": 0.04886169731616974, + "cosine_similarity": 0.9998109340667725, + "kl_divergence": 0.0013024145853326003, + "sigma_level": 64.2417199046864, + "cpk": 21.15232667880772, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json index 2e061ff232..1be80c540f 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "c642576eecb62daa", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md heading '2026-09-06T13:5xZ' \u2014 the record states the hour and deliberately not the minute, so only the date is carried here", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9986071586608887, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.7765593528747559, - "mean_abs_diff": 0.11612559109926224, - "cosine_similarity": 0.9986071586608887, - "kl_divergence": 0.011310374807126194, - "sigma_level": 27.6123564403692, - "cpk": 8.936910379016284, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.21668052673339844, - "mean_abs_diff": 0.03283289819955826, - "cosine_similarity": 0.9998907446861267, - "kl_divergence": 6.3402314768562945e-6, - "sigma_level": 96.19123042846297, - "cpk": 31.800557069791992, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.21230220794677734, - "mean_abs_diff": 0.036253008991479874, - "cosine_similarity": 0.9999125599861145, - "kl_divergence": 1.2138987899670097e-6, - "sigma_level": 87.78216091036128, - "cpk": 28.99552301438919, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.22992169857025146, - "mean_abs_diff": 0.031362131237983704, - "cosine_similarity": 0.9999329447746277, - "kl_divergence": 0.00009697837023952451, - "sigma_level": 99.8193545508211, - "cpk": 33.01223920881423, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.20900297164916992, - "mean_abs_diff": 0.03212369233369827, - "cosine_similarity": 0.9999116063117981, - "kl_divergence": 1.2130307390144058e-6, - "sigma_level": 97.33557518850837, - "cpk": 32.184626890296194, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2239065170288086, - "mean_abs_diff": 0.033580731600522995, - "cosine_similarity": 0.9999029040336609, - "kl_divergence": 0.00002739006113745167, - "sigma_level": 95.1876333412705, - "cpk": 31.462838583179984, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.26349353790283203, - "mean_abs_diff": 0.037349067628383636, - "cosine_similarity": 0.9999385476112366, - "kl_divergence": 6.901890175435653e-7, - "sigma_level": 85.80957537462608, - "cpk": 28.336116155389544, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.19062137603759766, - "mean_abs_diff": 0.03048071265220642, - "cosine_similarity": 0.9999557137489319, - "kl_divergence": 8.014137110110945e-7, - "sigma_level": 103.11592767396162, - "cpk": 34.11005531121256, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18280649185180664, - "mean_abs_diff": 0.02944883517920971, - "cosine_similarity": 0.9999102354049683, - "kl_divergence": 0.00036753254509921214, - "sigma_level": 106.09218564707784, - "cpk": 35.10370427494904, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2662510871887207, - "mean_abs_diff": 0.03750442713499069, - "cosine_similarity": 0.9999082684516907, - "kl_divergence": 0.0008122461170267515, - "sigma_level": 83.32380704426187, - "cpk": 27.514184710595494, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.21428704261779785, - "mean_abs_diff": 0.03618254512548447, - "cosine_similarity": 0.9999296069145203, - "kl_divergence": 0.0013414333957104756, - "sigma_level": 86.32583699113798, - "cpk": 28.514988289343737, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.2245645523071289, - "mean_abs_diff": 0.03425215184688568, - "cosine_similarity": 0.9999250173568726, - "kl_divergence": 0.0005234989769493335, - "sigma_level": 91.05669514978527, - "cpk": 30.09232440418292, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.24261903762817383, - "mean_abs_diff": 0.032688990235328674, - "cosine_similarity": 0.9999192953109741, - "kl_divergence": 0.00023210232981628998, - "sigma_level": 96.43691504783891, - "cpk": 31.882936234752634, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.19068384170532227, - "mean_abs_diff": 0.029661044478416443, - "cosine_similarity": 0.9999234080314636, - "kl_divergence": 0.00042034598196102077, - "sigma_level": 106.34280606113548, - "cpk": 35.184748795333576, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.22031402587890625, - "mean_abs_diff": 0.03211377561092377, - "cosine_similarity": 0.9999381899833679, - "kl_divergence": 0.00032284577336933807, - "sigma_level": 97.25797222535725, - "cpk": 32.15904735041753, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.26050472259521484, - "mean_abs_diff": 0.05659817159175873, - "cosine_similarity": 0.9997627139091492, - "kl_divergence": 0.0003138751302377445, - "sigma_level": 60.00271273002659, - "cpk": 19.71790059075344, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2220613956451416, - "mean_abs_diff": 0.034589797258377075, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.0010091621728843534, - "sigma_level": 90.97388370911588, - "cpk": 30.06239722026317, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.2452479600906372, - "mean_abs_diff": 0.04691462218761444, - "cosine_similarity": 0.9998961687088013, - "kl_divergence": 0.000010286648444546095, - "sigma_level": 71.21310678671547, - "cpk": 23.45929092892974, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.20011234283447266, - "mean_abs_diff": 0.028810735791921616, - "cosine_similarity": 0.9999183416366577, - "kl_divergence": 0.0005886006120770999, - "sigma_level": 109.42638674224135, - "cpk": 36.21274102098917, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 9904, - "gpu_argmax": 5942, - "max_abs_diff": 0.2104625701904297, - "mean_abs_diff": 0.03633837774395943, - "cosine_similarity": 0.9999009966850281, - "kl_divergence": 0.000780974228869925, - "sigma_level": 89.61552988240601, - "cpk": 29.60046971275267, - "verdict": "WarnArgmax" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.2549419403076172, - "mean_abs_diff": 0.03461437672376633, - "cosine_similarity": 0.9998999238014221, - "kl_divergence": 0.0005821044683649174, - "sigma_level": 90.29548641907701, - "cpk": 29.838035307745198, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.2721090316772461, - "mean_abs_diff": 0.041589464992284775, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.0012391113766812047, - "sigma_level": 75.04357270930153, - "cpk": 24.754439066426386, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.23730623722076416, - "mean_abs_diff": 0.040260475128889084, - "cosine_similarity": 0.9998979568481445, - "kl_divergence": 0.0009176755486284543, - "sigma_level": 79.46140806613252, - "cpk": 26.22053985178142, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19652128219604492, - "mean_abs_diff": 0.032911960035562515, - "cosine_similarity": 0.9999001026153564, - "kl_divergence": 0.00011332451760925956, - "sigma_level": 95.68482644088411, - "cpk": 31.632510881642027, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.1758362054824829, - "mean_abs_diff": 0.029195617884397507, - "cosine_similarity": 0.9998802542686462, - "kl_divergence": 0.0006681351605656909, - "sigma_level": 106.90541285517136, - "cpk": 35.37504015309934, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.18475735187530518, - "mean_abs_diff": 0.02708299830555916, - "cosine_similarity": 0.9998446702957153, - "kl_divergence": 0.00006509554974005071, - "sigma_level": 115.87629249589668, - "cpk": 38.363907712522156, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.17609357833862305, - "mean_abs_diff": 0.027387676760554314, - "cosine_similarity": 0.9998769164085388, - "kl_divergence": 0.00018923296147374256, - "sigma_level": 114.58697406979748, - "cpk": 37.93413543936636, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.20337700843811035, - "mean_abs_diff": 0.0357307530939579, - "cosine_similarity": 0.9997212886810303, - "kl_divergence": 0.00068673561845549, - "sigma_level": 89.93727787207827, - "cpk": 29.7112987348936, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.2158222198486328, - "mean_abs_diff": 0.03296241909265518, - "cosine_similarity": 0.9998617768287659, - "kl_divergence": 0.0007174528587839378, - "sigma_level": 94.28973702732381, - "cpk": 31.170910856772032, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.18802005052566528, - "mean_abs_diff": 0.030701307579874992, - "cosine_similarity": 0.9998907446861267, - "kl_divergence": 0.000727208578105642, - "sigma_level": 102.51328926047806, - "cpk": 33.90882208477513, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.18789291381835938, - "mean_abs_diff": 0.028557559475302696, - "cosine_similarity": 0.9998390674591064, - "kl_divergence": 0.00007215432969221762, - "sigma_level": 109.68622544018605, - "cpk": 36.30104423784289, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.1830120086669922, - "mean_abs_diff": 0.029256917536258698, - "cosine_similarity": 0.9998962879180908, - "kl_divergence": 0.0002161959497020309, - "sigma_level": 107.87296541749743, - "cpk": 35.69465260136486, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.18589067459106445, - "mean_abs_diff": 0.030194438993930817, - "cosine_similarity": 0.9999088644981384, - "kl_divergence": 0.0004802196691873215, - "sigma_level": 105.23769924870247, - "cpk": 34.81443364208194, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.19057416915893555, - "mean_abs_diff": 0.030395982787013054, - "cosine_similarity": 0.9998965263366699, - "kl_divergence": 0.0008087840754308433, - "sigma_level": 103.21133882875218, - "cpk": 34.14234543637878, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.20451754331588745, - "mean_abs_diff": 0.03417585417628288, - "cosine_similarity": 0.999851405620575, - "kl_divergence": 0.000011874646119087019, - "sigma_level": 93.7446562083357, - "cpk": 30.98123509441341, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.2111678123474121, - "mean_abs_diff": 0.031797345727682114, - "cosine_similarity": 0.9999288320541382, - "kl_divergence": 0.0003358035896646284, - "sigma_level": 98.93926386784285, - "cpk": 32.71758745767694, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.2316608428955078, - "mean_abs_diff": 0.03618849068880081, - "cosine_similarity": 0.999769926071167, - "kl_divergence": 0.0009839003068092891, - "sigma_level": 87.81894152240396, - "cpk": 29.008144261835998, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.21175193786621094, - "mean_abs_diff": 0.03807161748409271, - "cosine_similarity": 0.9998552203178406, - "kl_divergence": 0.00012716871719756028, - "sigma_level": 85.63607322735568, - "cpk": 28.273665757222588, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.22519683837890625, - "mean_abs_diff": 0.032994192093610764, - "cosine_similarity": 0.9999074339866638, - "kl_divergence": 0.001043569292129262, - "sigma_level": 94.46420366663784, - "cpk": 31.228337048733696, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.19043952226638794, - "mean_abs_diff": 0.030219942331314087, - "cosine_similarity": 0.999907910823822, - "kl_divergence": 0.0013804227424616085, - "sigma_level": 104.09966497830833, - "cpk": 34.437731170073306, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.19405746459960938, - "mean_abs_diff": 0.03200043737888336, - "cosine_similarity": 0.9999300837516785, - "kl_divergence": 0.0005307637452940784, - "sigma_level": 99.51118657903375, - "cpk": 32.90502873512619, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2977466583251953, - "mean_abs_diff": 0.04491940513253212, - "cosine_similarity": 0.9999076128005981, - "kl_divergence": 0.0019453823155301797, - "sigma_level": 71.66086038536118, - "cpk": 23.61870652680407, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.20647287368774414, - "mean_abs_diff": 0.029774188995361328, - "cosine_similarity": 0.9999259114265442, - "kl_divergence": 0.0004456308198784806, - "sigma_level": 105.13497563493686, - "cpk": 34.784132825430845, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.17754626274108887, - "mean_abs_diff": 0.028160495683550835, - "cosine_similarity": 0.9998858571052551, - "kl_divergence": 0.00016564224134654518, - "sigma_level": 111.07898880573893, - "cpk": 36.765659653179874, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.37882566452026367, - "mean_abs_diff": 0.0732780396938324, - "cosine_similarity": 0.9998091459274292, - "kl_divergence": 0.0011416456636240088, - "sigma_level": 44.864361894597046, - "cpk": 14.680822923886454, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.20982575416564941, - "mean_abs_diff": 0.031462863087654114, - "cosine_similarity": 0.9999397993087769, - "kl_divergence": 0.0005747938176134488, - "sigma_level": 99.36599836682235, - "cpk": 32.86147122209217, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.24659061431884766, - "mean_abs_diff": 0.04355722293257713, - "cosine_similarity": 0.9999262094497681, - "kl_divergence": 0.001248096566017884, - "sigma_level": 74.74092104068832, - "cpk": 24.642348100233157, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.21674823760986328, - "mean_abs_diff": 0.030111797153949738, - "cosine_similarity": 0.9999454617500305, - "kl_divergence": 0.0008462897816879462, - "sigma_level": 103.61291944976223, - "cpk": 34.277642215504095, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1926441192626953, - "mean_abs_diff": 0.033761173486709595, - "cosine_similarity": 0.9999311566352844, - "kl_divergence": 0.0010025746520832025, - "sigma_level": 94.19188658464856, - "cpk": 31.132293142880787, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2571578025817871, - "mean_abs_diff": 0.04913201928138733, - "cosine_similarity": 0.9998971223831177, - "kl_divergence": 0.00016394181358259913, - "sigma_level": 66.67619384362449, - "cpk": 21.952403277746956, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.20004218816757202, - "mean_abs_diff": 0.032239872962236404, - "cosine_similarity": 0.9999257326126099, - "kl_divergence": 0.00035964655689502595, - "sigma_level": 98.03535416893781, - "cpk": 32.41506410929474, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.22790932655334473, - "mean_abs_diff": 0.032669227570295334, - "cosine_similarity": 0.9999305009841919, - "kl_divergence": 0.00022142489203358796, - "sigma_level": 96.35532470982132, - "cpk": 31.856120400727534, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.20201587677001953, - "mean_abs_diff": 0.029387902468442917, - "cosine_similarity": 0.9999483227729797, - "kl_divergence": 0.0004107862516253817, - "sigma_level": 106.90799937791267, - "cpk": 35.374182971069686, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2490062713623047, - "mean_abs_diff": 0.03013196960091591, - "cosine_similarity": 0.999924898147583, - "kl_divergence": 0.00016009342785522766, - "sigma_level": 103.912201916083, - "cpk": 34.37647736291937, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19240188598632812, - "mean_abs_diff": 0.02893163077533245, - "cosine_similarity": 0.9999112486839294, - "kl_divergence": 6.817382872259671e-6, - "sigma_level": 108.02736989205229, - "cpk": 35.748672632405174, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.267974853515625, - "mean_abs_diff": 0.03601562976837158, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.00007162719322224682, - "sigma_level": 88.31999199829174, - "cpk": 29.174922321684246, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.24425232410430908, - "mean_abs_diff": 0.033856507390737534, - "cosine_similarity": 0.9999051690101624, - "kl_divergence": 0.0010536469720982588, - "sigma_level": 92.8341216476468, - "cpk": 30.682787288742585, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.20838356018066406, - "mean_abs_diff": 0.036430489271879196, - "cosine_similarity": 0.9999008774757385, - "kl_divergence": 0.0004580104239343563, - "sigma_level": 88.14959348258607, - "cpk": 29.115586759221365, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.21566104888916016, - "mean_abs_diff": 0.035551492124795914, - "cosine_similarity": 0.9999303221702576, - "kl_divergence": 0.00005223383602209498, - "sigma_level": 89.79387134077257, - "cpk": 29.66526493777199, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.19229507446289062, - "mean_abs_diff": 0.029516855254769325, - "cosine_similarity": 0.9999398589134216, - "kl_divergence": 0.00006387025116334416, - "sigma_level": 105.89702480105015, - "cpk": 35.03852933760311, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.21092748641967773, - "mean_abs_diff": 0.034741658717393875, - "cosine_similarity": 0.9999397993087769, - "kl_divergence": 0.000672524288997609, - "sigma_level": 92.15167487319876, - "cpk": 30.45043312117618, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2311105728149414, - "mean_abs_diff": 0.037915751338005066, - "cosine_similarity": 0.999920666217804, - "kl_divergence": 0.0001797125477679641, - "sigma_level": 84.79673733628577, - "cpk": 27.99765144483553, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.23068904876708984, - "mean_abs_diff": 0.0350475087761879, - "cosine_similarity": 0.9999470114707947, - "kl_divergence": 0.0010822221598805622, - "sigma_level": 91.749185936065, - "cpk": 30.315096945413142, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.17993736267089844, - "mean_abs_diff": 0.028637047857046127, - "cosine_similarity": 0.9999294877052307, - "kl_divergence": 0.00021486480617655506, - "sigma_level": 110.01873575701202, - "cpk": 36.41036093558357, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2231612205505371, - "mean_abs_diff": 0.04158709943294525, - "cosine_similarity": 0.9999403953552246, - "kl_divergence": 0.0008206746305046884, - "sigma_level": 78.4256339651499, - "cpk": 25.870086768566598, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.1945490837097168, - "mean_abs_diff": 0.03391721844673157, - "cosine_similarity": 0.9999347925186157, - "kl_divergence": 0.00013648398155169364, - "sigma_level": 95.6293235804782, - "cpk": 31.60615113867671, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18570899963378906, - "mean_abs_diff": 0.02739095687866211, - "cosine_similarity": 0.999912440776825, - "kl_divergence": 0.0006194080312011955, - "sigma_level": 114.51598314013269, - "cpk": 37.91060251703515, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.2039031982421875, - "mean_abs_diff": 0.029785331338644028, - "cosine_similarity": 0.9999380707740784, - "kl_divergence": 0.0008764585761620313, - "sigma_level": 105.47572482880632, - "cpk": 34.89677249191797, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.19327348470687866, - "mean_abs_diff": 0.0273088701069355, - "cosine_similarity": 0.9999405741691589, - "kl_divergence": 0.0003229069364669503, - "sigma_level": 115.07232534245732, - "cpk": 38.095567182012424, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.20456242561340332, - "mean_abs_diff": 0.03238324820995331, - "cosine_similarity": 0.9999169707298279, - "kl_divergence": 9.455850442395312e-6, - "sigma_level": 99.50280412389094, - "cpk": 32.89908270766946, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.2868785858154297, - "mean_abs_diff": 0.05693440139293671, - "cosine_similarity": 0.9998724460601807, - "kl_divergence": 0.0003243814681716116, - "sigma_level": 58.33631463373317, - "cpk": 19.168659615149256, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.25117525458335876, - "mean_abs_diff": 0.03555203229188919, - "cosine_similarity": 0.9999099969863892, - "kl_divergence": 0.0003880159170877742, - "sigma_level": 89.1645369985614, - "cpk": 29.45734729129845, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.28829193115234375, - "mean_abs_diff": 0.041850414127111435, - "cosine_similarity": 0.9998987317085266, - "kl_divergence": 0.0014878545190500573, - "sigma_level": 76.81169855905522, - "cpk": 25.336016070143128, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2536201477050781, - "mean_abs_diff": 0.03538894280791283, - "cosine_similarity": 0.9999024271965027, - "kl_divergence": 0.00042057384320396996, - "sigma_level": 90.20118474225366, - "cpk": 29.801051200080423, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.21267127990722656, - "mean_abs_diff": 0.034602489322423935, - "cosine_similarity": 0.9999219179153442, - "kl_divergence": 0.0017517055542956828, - "sigma_level": 92.15392100647352, - "cpk": 30.452244079854008, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.24674510955810547, - "mean_abs_diff": 0.028482820838689804, - "cosine_similarity": 0.9999575614929199, - "kl_divergence": 0.0009479499398340706, - "sigma_level": 109.40058760407271, - "cpk": 36.20719275665972, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2071094512939453, - "mean_abs_diff": 0.03381306678056717, - "cosine_similarity": 0.9999055862426758, - "kl_divergence": 0.0005057522906840319, - "sigma_level": 94.66239531405454, - "cpk": 31.287396280154628, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.2892899513244629, - "mean_abs_diff": 0.030646586790680885, - "cosine_similarity": 0.9998894929885864, - "kl_divergence": 0.0005098066796168379, - "sigma_level": 102.54242438017586, - "cpk": 33.91892685101747, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.7765593528747559, + "mean_abs_diff": 0.11612559109926224, + "cosine_similarity": 0.9986071586608887, + "kl_divergence": 0.011310374807126194, + "sigma_level": 27.6123564403692, + "cpk": 8.936910379016284, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.21668052673339844, + "mean_abs_diff": 0.03283289819955826, + "cosine_similarity": 0.9998907446861267, + "kl_divergence": 6.3402314768562945e-06, + "sigma_level": 96.19123042846297, + "cpk": 31.800557069791992, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.21230220794677734, + "mean_abs_diff": 0.036253008991479874, + "cosine_similarity": 0.9999125599861145, + "kl_divergence": 1.2138987899670097e-06, + "sigma_level": 87.78216091036128, + "cpk": 28.99552301438919, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.22992169857025146, + "mean_abs_diff": 0.031362131237983704, + "cosine_similarity": 0.9999329447746277, + "kl_divergence": 9.697837023952451e-05, + "sigma_level": 99.8193545508211, + "cpk": 33.01223920881423, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.20900297164916992, + "mean_abs_diff": 0.03212369233369827, + "cosine_similarity": 0.9999116063117981, + "kl_divergence": 1.2130307390144058e-06, + "sigma_level": 97.33557518850837, + "cpk": 32.184626890296194, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2239065170288086, + "mean_abs_diff": 0.033580731600522995, + "cosine_similarity": 0.9999029040336609, + "kl_divergence": 2.739006113745167e-05, + "sigma_level": 95.1876333412705, + "cpk": 31.462838583179984, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.26349353790283203, + "mean_abs_diff": 0.037349067628383636, + "cosine_similarity": 0.9999385476112366, + "kl_divergence": 6.901890175435653e-07, + "sigma_level": 85.80957537462608, + "cpk": 28.336116155389544, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.19062137603759766, + "mean_abs_diff": 0.03048071265220642, + "cosine_similarity": 0.9999557137489319, + "kl_divergence": 8.014137110110945e-07, + "sigma_level": 103.11592767396162, + "cpk": 34.11005531121256, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18280649185180664, + "mean_abs_diff": 0.02944883517920971, + "cosine_similarity": 0.9999102354049683, + "kl_divergence": 0.00036753254509921214, + "sigma_level": 106.09218564707784, + "cpk": 35.10370427494904, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2662510871887207, + "mean_abs_diff": 0.03750442713499069, + "cosine_similarity": 0.9999082684516907, + "kl_divergence": 0.0008122461170267515, + "sigma_level": 83.32380704426187, + "cpk": 27.514184710595494, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.21428704261779785, + "mean_abs_diff": 0.03618254512548447, + "cosine_similarity": 0.9999296069145203, + "kl_divergence": 0.0013414333957104756, + "sigma_level": 86.32583699113798, + "cpk": 28.514988289343737, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.2245645523071289, + "mean_abs_diff": 0.03425215184688568, + "cosine_similarity": 0.9999250173568726, + "kl_divergence": 0.0005234989769493335, + "sigma_level": 91.05669514978527, + "cpk": 30.09232440418292, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.24261903762817383, + "mean_abs_diff": 0.032688990235328674, + "cosine_similarity": 0.9999192953109741, + "kl_divergence": 0.00023210232981628998, + "sigma_level": 96.43691504783891, + "cpk": 31.882936234752634, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.19068384170532227, + "mean_abs_diff": 0.029661044478416443, + "cosine_similarity": 0.9999234080314636, + "kl_divergence": 0.00042034598196102077, + "sigma_level": 106.34280606113548, + "cpk": 35.184748795333576, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.22031402587890625, + "mean_abs_diff": 0.03211377561092377, + "cosine_similarity": 0.9999381899833679, + "kl_divergence": 0.00032284577336933807, + "sigma_level": 97.25797222535725, + "cpk": 32.15904735041753, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.26050472259521484, + "mean_abs_diff": 0.05659817159175873, + "cosine_similarity": 0.9997627139091492, + "kl_divergence": 0.0003138751302377445, + "sigma_level": 60.00271273002659, + "cpk": 19.71790059075344, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2220613956451416, + "mean_abs_diff": 0.034589797258377075, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 0.0010091621728843534, + "sigma_level": 90.97388370911588, + "cpk": 30.06239722026317, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.2452479600906372, + "mean_abs_diff": 0.04691462218761444, + "cosine_similarity": 0.9998961687088013, + "kl_divergence": 1.0286648444546095e-05, + "sigma_level": 71.21310678671547, + "cpk": 23.45929092892974, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.20011234283447266, + "mean_abs_diff": 0.028810735791921616, + "cosine_similarity": 0.9999183416366577, + "kl_divergence": 0.0005886006120770999, + "sigma_level": 109.42638674224135, + "cpk": 36.21274102098917, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 9904, + "gpu_argmax": 5942, + "max_abs_diff": 0.2104625701904297, + "mean_abs_diff": 0.03633837774395943, + "cosine_similarity": 0.9999009966850281, + "kl_divergence": 0.000780974228869925, + "sigma_level": 89.61552988240601, + "cpk": 29.60046971275267, + "verdict": "WarnArgmax" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.2549419403076172, + "mean_abs_diff": 0.03461437672376633, + "cosine_similarity": 0.9998999238014221, + "kl_divergence": 0.0005821044683649174, + "sigma_level": 90.29548641907701, + "cpk": 29.838035307745198, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.2721090316772461, + "mean_abs_diff": 0.041589464992284775, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.0012391113766812047, + "sigma_level": 75.04357270930153, + "cpk": 24.754439066426386, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.23730623722076416, + "mean_abs_diff": 0.040260475128889084, + "cosine_similarity": 0.9998979568481445, + "kl_divergence": 0.0009176755486284543, + "sigma_level": 79.46140806613252, + "cpk": 26.22053985178142, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19652128219604492, + "mean_abs_diff": 0.032911960035562515, + "cosine_similarity": 0.9999001026153564, + "kl_divergence": 0.00011332451760925956, + "sigma_level": 95.68482644088411, + "cpk": 31.632510881642027, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.1758362054824829, + "mean_abs_diff": 0.029195617884397507, + "cosine_similarity": 0.9998802542686462, + "kl_divergence": 0.0006681351605656909, + "sigma_level": 106.90541285517136, + "cpk": 35.37504015309934, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.18475735187530518, + "mean_abs_diff": 0.02708299830555916, + "cosine_similarity": 0.9998446702957153, + "kl_divergence": 6.509554974005071e-05, + "sigma_level": 115.87629249589668, + "cpk": 38.363907712522156, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.17609357833862305, + "mean_abs_diff": 0.027387676760554314, + "cosine_similarity": 0.9998769164085388, + "kl_divergence": 0.00018923296147374256, + "sigma_level": 114.58697406979748, + "cpk": 37.93413543936636, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.20337700843811035, + "mean_abs_diff": 0.0357307530939579, + "cosine_similarity": 0.9997212886810303, + "kl_divergence": 0.00068673561845549, + "sigma_level": 89.93727787207827, + "cpk": 29.7112987348936, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.2158222198486328, + "mean_abs_diff": 0.03296241909265518, + "cosine_similarity": 0.9998617768287659, + "kl_divergence": 0.0007174528587839378, + "sigma_level": 94.28973702732381, + "cpk": 31.170910856772032, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.18802005052566528, + "mean_abs_diff": 0.030701307579874992, + "cosine_similarity": 0.9998907446861267, + "kl_divergence": 0.000727208578105642, + "sigma_level": 102.51328926047806, + "cpk": 33.90882208477513, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.18789291381835938, + "mean_abs_diff": 0.028557559475302696, + "cosine_similarity": 0.9998390674591064, + "kl_divergence": 7.215432969221762e-05, + "sigma_level": 109.68622544018605, + "cpk": 36.30104423784289, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.1830120086669922, + "mean_abs_diff": 0.029256917536258698, + "cosine_similarity": 0.9998962879180908, + "kl_divergence": 0.0002161959497020309, + "sigma_level": 107.87296541749743, + "cpk": 35.69465260136486, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.18589067459106445, + "mean_abs_diff": 0.030194438993930817, + "cosine_similarity": 0.9999088644981384, + "kl_divergence": 0.0004802196691873215, + "sigma_level": 105.23769924870247, + "cpk": 34.81443364208194, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.19057416915893555, + "mean_abs_diff": 0.030395982787013054, + "cosine_similarity": 0.9998965263366699, + "kl_divergence": 0.0008087840754308433, + "sigma_level": 103.21133882875218, + "cpk": 34.14234543637878, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.20451754331588745, + "mean_abs_diff": 0.03417585417628288, + "cosine_similarity": 0.999851405620575, + "kl_divergence": 1.1874646119087019e-05, + "sigma_level": 93.7446562083357, + "cpk": 30.98123509441341, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.2111678123474121, + "mean_abs_diff": 0.031797345727682114, + "cosine_similarity": 0.9999288320541382, + "kl_divergence": 0.0003358035896646284, + "sigma_level": 98.93926386784285, + "cpk": 32.71758745767694, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.2316608428955078, + "mean_abs_diff": 0.03618849068880081, + "cosine_similarity": 0.999769926071167, + "kl_divergence": 0.0009839003068092891, + "sigma_level": 87.81894152240396, + "cpk": 29.008144261835998, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.21175193786621094, + "mean_abs_diff": 0.03807161748409271, + "cosine_similarity": 0.9998552203178406, + "kl_divergence": 0.00012716871719756028, + "sigma_level": 85.63607322735568, + "cpk": 28.273665757222588, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.22519683837890625, + "mean_abs_diff": 0.032994192093610764, + "cosine_similarity": 0.9999074339866638, + "kl_divergence": 0.001043569292129262, + "sigma_level": 94.46420366663784, + "cpk": 31.228337048733696, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.19043952226638794, + "mean_abs_diff": 0.030219942331314087, + "cosine_similarity": 0.999907910823822, + "kl_divergence": 0.0013804227424616085, + "sigma_level": 104.09966497830833, + "cpk": 34.437731170073306, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.19405746459960938, + "mean_abs_diff": 0.03200043737888336, + "cosine_similarity": 0.9999300837516785, + "kl_divergence": 0.0005307637452940784, + "sigma_level": 99.51118657903375, + "cpk": 32.90502873512619, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2977466583251953, + "mean_abs_diff": 0.04491940513253212, + "cosine_similarity": 0.9999076128005981, + "kl_divergence": 0.0019453823155301797, + "sigma_level": 71.66086038536118, + "cpk": 23.61870652680407, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.20647287368774414, + "mean_abs_diff": 0.029774188995361328, + "cosine_similarity": 0.9999259114265442, + "kl_divergence": 0.0004456308198784806, + "sigma_level": 105.13497563493686, + "cpk": 34.784132825430845, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.17754626274108887, + "mean_abs_diff": 0.028160495683550835, + "cosine_similarity": 0.9998858571052551, + "kl_divergence": 0.00016564224134654518, + "sigma_level": 111.07898880573893, + "cpk": 36.765659653179874, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.37882566452026367, + "mean_abs_diff": 0.0732780396938324, + "cosine_similarity": 0.9998091459274292, + "kl_divergence": 0.0011416456636240088, + "sigma_level": 44.864361894597046, + "cpk": 14.680822923886454, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.20982575416564941, + "mean_abs_diff": 0.031462863087654114, + "cosine_similarity": 0.9999397993087769, + "kl_divergence": 0.0005747938176134488, + "sigma_level": 99.36599836682235, + "cpk": 32.86147122209217, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.24659061431884766, + "mean_abs_diff": 0.04355722293257713, + "cosine_similarity": 0.9999262094497681, + "kl_divergence": 0.001248096566017884, + "sigma_level": 74.74092104068832, + "cpk": 24.642348100233157, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.21674823760986328, + "mean_abs_diff": 0.030111797153949738, + "cosine_similarity": 0.9999454617500305, + "kl_divergence": 0.0008462897816879462, + "sigma_level": 103.61291944976223, + "cpk": 34.277642215504095, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1926441192626953, + "mean_abs_diff": 0.033761173486709595, + "cosine_similarity": 0.9999311566352844, + "kl_divergence": 0.0010025746520832025, + "sigma_level": 94.19188658464856, + "cpk": 31.132293142880787, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2571578025817871, + "mean_abs_diff": 0.04913201928138733, + "cosine_similarity": 0.9998971223831177, + "kl_divergence": 0.00016394181358259913, + "sigma_level": 66.67619384362449, + "cpk": 21.952403277746956, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.20004218816757202, + "mean_abs_diff": 0.032239872962236404, + "cosine_similarity": 0.9999257326126099, + "kl_divergence": 0.00035964655689502595, + "sigma_level": 98.03535416893781, + "cpk": 32.41506410929474, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.22790932655334473, + "mean_abs_diff": 0.032669227570295334, + "cosine_similarity": 0.9999305009841919, + "kl_divergence": 0.00022142489203358796, + "sigma_level": 96.35532470982132, + "cpk": 31.856120400727534, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.20201587677001953, + "mean_abs_diff": 0.029387902468442917, + "cosine_similarity": 0.9999483227729797, + "kl_divergence": 0.0004107862516253817, + "sigma_level": 106.90799937791267, + "cpk": 35.374182971069686, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2490062713623047, + "mean_abs_diff": 0.03013196960091591, + "cosine_similarity": 0.999924898147583, + "kl_divergence": 0.00016009342785522766, + "sigma_level": 103.912201916083, + "cpk": 34.37647736291937, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19240188598632812, + "mean_abs_diff": 0.02893163077533245, + "cosine_similarity": 0.9999112486839294, + "kl_divergence": 6.817382872259671e-06, + "sigma_level": 108.02736989205229, + "cpk": 35.748672632405174, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.267974853515625, + "mean_abs_diff": 0.03601562976837158, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 7.162719322224682e-05, + "sigma_level": 88.31999199829174, + "cpk": 29.174922321684246, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.24425232410430908, + "mean_abs_diff": 0.033856507390737534, + "cosine_similarity": 0.9999051690101624, + "kl_divergence": 0.0010536469720982588, + "sigma_level": 92.8341216476468, + "cpk": 30.682787288742585, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.20838356018066406, + "mean_abs_diff": 0.036430489271879196, + "cosine_similarity": 0.9999008774757385, + "kl_divergence": 0.0004580104239343563, + "sigma_level": 88.14959348258607, + "cpk": 29.115586759221365, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.21566104888916016, + "mean_abs_diff": 0.035551492124795914, + "cosine_similarity": 0.9999303221702576, + "kl_divergence": 5.223383602209498e-05, + "sigma_level": 89.79387134077257, + "cpk": 29.66526493777199, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.19229507446289062, + "mean_abs_diff": 0.029516855254769325, + "cosine_similarity": 0.9999398589134216, + "kl_divergence": 6.387025116334416e-05, + "sigma_level": 105.89702480105015, + "cpk": 35.03852933760311, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.21092748641967773, + "mean_abs_diff": 0.034741658717393875, + "cosine_similarity": 0.9999397993087769, + "kl_divergence": 0.000672524288997609, + "sigma_level": 92.15167487319876, + "cpk": 30.45043312117618, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2311105728149414, + "mean_abs_diff": 0.037915751338005066, + "cosine_similarity": 0.999920666217804, + "kl_divergence": 0.0001797125477679641, + "sigma_level": 84.79673733628577, + "cpk": 27.99765144483553, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.23068904876708984, + "mean_abs_diff": 0.0350475087761879, + "cosine_similarity": 0.9999470114707947, + "kl_divergence": 0.0010822221598805622, + "sigma_level": 91.749185936065, + "cpk": 30.315096945413142, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.17993736267089844, + "mean_abs_diff": 0.028637047857046127, + "cosine_similarity": 0.9999294877052307, + "kl_divergence": 0.00021486480617655506, + "sigma_level": 110.01873575701202, + "cpk": 36.41036093558357, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2231612205505371, + "mean_abs_diff": 0.04158709943294525, + "cosine_similarity": 0.9999403953552246, + "kl_divergence": 0.0008206746305046884, + "sigma_level": 78.4256339651499, + "cpk": 25.870086768566598, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.1945490837097168, + "mean_abs_diff": 0.03391721844673157, + "cosine_similarity": 0.9999347925186157, + "kl_divergence": 0.00013648398155169364, + "sigma_level": 95.6293235804782, + "cpk": 31.60615113867671, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18570899963378906, + "mean_abs_diff": 0.02739095687866211, + "cosine_similarity": 0.999912440776825, + "kl_divergence": 0.0006194080312011955, + "sigma_level": 114.51598314013269, + "cpk": 37.91060251703515, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.2039031982421875, + "mean_abs_diff": 0.029785331338644028, + "cosine_similarity": 0.9999380707740784, + "kl_divergence": 0.0008764585761620313, + "sigma_level": 105.47572482880632, + "cpk": 34.89677249191797, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.19327348470687866, + "mean_abs_diff": 0.0273088701069355, + "cosine_similarity": 0.9999405741691589, + "kl_divergence": 0.0003229069364669503, + "sigma_level": 115.07232534245732, + "cpk": 38.095567182012424, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.20456242561340332, + "mean_abs_diff": 0.03238324820995331, + "cosine_similarity": 0.9999169707298279, + "kl_divergence": 9.455850442395312e-06, + "sigma_level": 99.50280412389094, + "cpk": 32.89908270766946, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.2868785858154297, + "mean_abs_diff": 0.05693440139293671, + "cosine_similarity": 0.9998724460601807, + "kl_divergence": 0.0003243814681716116, + "sigma_level": 58.33631463373317, + "cpk": 19.168659615149256, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.25117525458335876, + "mean_abs_diff": 0.03555203229188919, + "cosine_similarity": 0.9999099969863892, + "kl_divergence": 0.0003880159170877742, + "sigma_level": 89.1645369985614, + "cpk": 29.45734729129845, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.28829193115234375, + "mean_abs_diff": 0.041850414127111435, + "cosine_similarity": 0.9998987317085266, + "kl_divergence": 0.0014878545190500573, + "sigma_level": 76.81169855905522, + "cpk": 25.336016070143128, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2536201477050781, + "mean_abs_diff": 0.03538894280791283, + "cosine_similarity": 0.9999024271965027, + "kl_divergence": 0.00042057384320396996, + "sigma_level": 90.20118474225366, + "cpk": 29.801051200080423, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.21267127990722656, + "mean_abs_diff": 0.034602489322423935, + "cosine_similarity": 0.9999219179153442, + "kl_divergence": 0.0017517055542956828, + "sigma_level": 92.15392100647352, + "cpk": 30.452244079854008, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.24674510955810547, + "mean_abs_diff": 0.028482820838689804, + "cosine_similarity": 0.9999575614929199, + "kl_divergence": 0.0009479499398340706, + "sigma_level": 109.40058760407271, + "cpk": 36.20719275665972, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2071094512939453, + "mean_abs_diff": 0.03381306678056717, + "cosine_similarity": 0.9999055862426758, + "kl_divergence": 0.0005057522906840319, + "sigma_level": 94.66239531405454, + "cpk": 31.287396280154628, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.2892899513244629, + "mean_abs_diff": 0.030646586790680885, + "cosine_similarity": 0.9998894929885864, + "kl_divergence": 0.0005098066796168379, + "sigma_level": 102.54242438017586, + "cpk": 33.91892685101747, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json index c29ba09d72..5d3189d2e7 100644 --- a/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "8cdb7c1a668127db", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md names the binary but carries no timestamp; the date is the git author date of the commit that ADDED this record \u2014 an upper bound on when it was measured, stated as such and not as a measurement", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9995829463005066, + "min_cosine_position": 22, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 16, - "gpu_argmax": 16, - "max_abs_diff": 0.17002153396606445, - "mean_abs_diff": 0.028188295662403107, - "cosine_similarity": 0.9999852776527405, - "kl_divergence": 0.0005291031695536818, - "sigma_level": 113.51094745441767, - "cpk": 37.57034247249217, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.31151044368743896, - "mean_abs_diff": 0.04776180535554886, - "cosine_similarity": 0.999913215637207, - "kl_divergence": 0.000018919660545105944, - "sigma_level": 65.37697481827193, - "cpk": 21.532114743923568, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.2960681915283203, - "mean_abs_diff": 0.05043606460094452, - "cosine_similarity": 0.9999374151229858, - "kl_divergence": 5.5291439027043675e-6, - "sigma_level": 61.84778951581121, - "cpk": 20.355983246316647, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.2933235168457031, - "mean_abs_diff": 0.04766261205077171, - "cosine_similarity": 0.9999246001243591, - "kl_divergence": 0.0001060628215343356, - "sigma_level": 66.19522509167993, - "cpk": 21.80215525279679, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.3401813507080078, - "mean_abs_diff": 0.05537569522857666, - "cosine_similarity": 0.9999253749847412, - "kl_divergence": 3.1806382760983147e-6, - "sigma_level": 58.14417981939996, - "cpk": 19.113078741383767, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.29981279373168945, - "mean_abs_diff": 0.04475972056388855, - "cosine_similarity": 0.9999157786369324, - "kl_divergence": 0.00006075290117685989, - "sigma_level": 70.78162520727061, - "cpk": 23.32986125531226, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.36730432510375977, - "mean_abs_diff": 0.05289534851908684, - "cosine_similarity": 0.9999330639839172, - "kl_divergence": 3.942939467082102e-6, - "sigma_level": 59.221911860577734, - "cpk": 19.47959031453992, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.37870264053344727, - "mean_abs_diff": 0.06961360573768616, - "cosine_similarity": 0.9999417066574097, - "kl_divergence": 5.816292943734862e-6, - "sigma_level": 47.307308057754874, - "cpk": 15.494666661614641, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2391514778137207, - "mean_abs_diff": 0.03515106812119484, - "cosine_similarity": 0.9999060034751892, - "kl_divergence": 0.00020459017940060442, - "sigma_level": 89.95740355697005, - "cpk": 29.72229295062028, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.2996964454650879, - "mean_abs_diff": 0.04814895987510681, - "cosine_similarity": 0.9999247193336487, - "kl_divergence": 0.0008225811496027273, - "sigma_level": 66.99980088661925, - "cpk": 22.064436068495585, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.2887105941772461, - "mean_abs_diff": 0.05602011829614639, - "cosine_similarity": 0.9999487400054932, - "kl_divergence": 0.0017507166523780418, - "sigma_level": 58.404129958133645, - "cpk": 19.19539279694137, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.23575425148010254, - "mean_abs_diff": 0.03749267756938934, - "cosine_similarity": 0.9999302625656128, - "kl_divergence": 0.0009205649545698543, - "sigma_level": 83.41319884576228, - "cpk": 27.5437842678078, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.23073053359985352, - "mean_abs_diff": 0.036286987364292145, - "cosine_similarity": 0.9999309778213501, - "kl_divergence": 0.0002103350091946032, - "sigma_level": 87.46254652146891, - "cpk": 28.889702813783522, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.22650158405303955, - "mean_abs_diff": 0.03817339614033699, - "cosine_similarity": 0.9999290108680725, - "kl_divergence": 0.0006199522697201657, - "sigma_level": 82.87208180962669, - "cpk": 27.360401535884456, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.2506117820739746, - "mean_abs_diff": 0.04409490525722504, - "cosine_similarity": 0.9999329447746277, - "kl_divergence": 0.001207147840417756, - "sigma_level": 72.75087140793437, - "cpk": 23.982961904135337, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2462446689605713, - "mean_abs_diff": 0.03615713492035866, - "cosine_similarity": 0.9999027252197266, - "kl_divergence": 0.0007957232607366745, - "sigma_level": 86.4713156254837, - "cpk": 28.563225623010272, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.30622434616088867, - "mean_abs_diff": 0.04285147041082382, - "cosine_similarity": 0.99992436170578, - "kl_divergence": 0.001562816857886753, - "sigma_level": 73.57721644186985, - "cpk": 24.262997821184154, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.26041460037231445, - "mean_abs_diff": 0.041633863002061844, - "cosine_similarity": 0.9999330043792725, - "kl_divergence": 0.0000386192979878687, - "sigma_level": 76.17994851863548, - "cpk": 25.129010711201076, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.2938199043273926, - "mean_abs_diff": 0.04622060805559158, - "cosine_similarity": 0.9999043345451355, - "kl_divergence": 0.0004776241426479791, - "sigma_level": 69.31179649604657, - "cpk": 22.836962717059464, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.33150339126586914, - "mean_abs_diff": 0.05491165816783905, - "cosine_similarity": 0.9998782873153687, - "kl_divergence": 0.004729332624206632, - "sigma_level": 59.30963249897647, - "cpk": 19.498478310838493, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.2719573974609375, - "mean_abs_diff": 0.0452549010515213, - "cosine_similarity": 0.9998787045478821, - "kl_divergence": 0.001985488312073216, - "sigma_level": 69.17673829662412, - "cpk": 22.798030561651316, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.5426011085510254, - "mean_abs_diff": 0.08933572471141815, - "cosine_similarity": 0.9997243881225586, - "kl_divergence": 0.0025612063383940826, - "sigma_level": 35.23386750744257, - "cpk": 11.482318911800565, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.6198992729187012, - "mean_abs_diff": 0.09157062321901321, - "cosine_similarity": 0.9995829463005066, - "kl_divergence": 0.010837422875741744, - "sigma_level": 34.05332167588209, - "cpk": 11.091250234582526, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.6332998275756836, - "mean_abs_diff": 0.0804496556520462, - "cosine_similarity": 0.9996832013130188, - "kl_divergence": 0.002525993250052904, - "sigma_level": 38.54398452241279, - "cpk": 12.589590650613772, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.35448646545410156, - "mean_abs_diff": 0.05539744719862938, - "cosine_similarity": 0.9998620748519897, - "kl_divergence": 0.002686264662680092, - "sigma_level": 56.62337561525491, - "cpk": 18.613059333347113, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3623623847961426, - "mean_abs_diff": 0.05665629357099533, - "cosine_similarity": 0.9997888207435608, - "kl_divergence": 0.00006107804999826922, - "sigma_level": 57.59732769966832, - "cpk": 18.927171640968005, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.26375818252563477, - "mean_abs_diff": 0.04152446612715721, - "cosine_similarity": 0.9999150633811951, - "kl_divergence": 0.00028931538420952634, - "sigma_level": 75.91556809492718, - "cpk": 25.042493245318916, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.298952579498291, - "mean_abs_diff": 0.05040597915649414, - "cosine_similarity": 0.9997300505638123, - "kl_divergence": 0.00029820851684663813, - "sigma_level": 63.07446362066332, - "cpk": 20.759877032006923, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.335299015045166, - "mean_abs_diff": 0.06050831079483032, - "cosine_similarity": 0.9998612999916077, - "kl_divergence": 0.002278695086270758, - "sigma_level": 52.97888535424999, - "cpk": 17.392489879701778, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3436293601989746, - "mean_abs_diff": 0.05135822668671608, - "cosine_similarity": 0.9998526573181152, - "kl_divergence": 0.001024066884138274, - "sigma_level": 61.078798535758686, - "cpk": 20.098191280173584, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2161470353603363, - "mean_abs_diff": 0.03684602305293083, - "cosine_similarity": 0.9998604655265808, - "kl_divergence": 0.000037301007973426357, - "sigma_level": 84.97813534510328, - "cpk": 28.0651195872077, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3007354736328125, - "mean_abs_diff": 0.05120687186717987, - "cosine_similarity": 0.9999154806137085, - "kl_divergence": 0.0007108443470111474, - "sigma_level": 63.124975763940014, - "cpk": 20.772289209183093, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.28400611877441406, - "mean_abs_diff": 0.048652347177267075, - "cosine_similarity": 0.999927282333374, - "kl_divergence": 0.0015689714082204687, - "sigma_level": 65.72234032440733, - "cpk": 21.640984598238646, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 1075, - "gpu_argmax": 1075, - "max_abs_diff": 0.2669934034347534, - "mean_abs_diff": 0.04793161153793335, - "cosine_similarity": 0.9999324083328247, - "kl_divergence": 0.0025805480581009115, - "sigma_level": 67.08375790360458, - "cpk": 22.09329991583983, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.23727846145629883, - "mean_abs_diff": 0.0352792851626873, - "cosine_similarity": 0.9998976588249207, - "kl_divergence": 0.00003295202382643496, - "sigma_level": 89.6666882397245, - "cpk": 29.625281357907912, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.33716487884521484, - "mean_abs_diff": 0.05362644046545029, - "cosine_similarity": 0.9999046921730042, - "kl_divergence": 0.004216564679063551, - "sigma_level": 58.67804631412172, - "cpk": 19.297124208266144, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.3685007095336914, - "mean_abs_diff": 0.06397877633571625, - "cosine_similarity": 0.9996156692504883, - "kl_divergence": 0.005635637283317798, - "sigma_level": 50.13709882194314, - "cpk": 16.445057088010145, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.2645277976989746, - "mean_abs_diff": 0.034263480454683304, - "cosine_similarity": 0.9998966455459595, - "kl_divergence": 0.00022884919989930267, - "sigma_level": 91.22137614961048, - "cpk": 30.146661896640868, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.34772682189941406, - "mean_abs_diff": 0.0546145886182785, - "cosine_similarity": 0.9999197125434875, - "kl_divergence": 0.0028962852285730335, - "sigma_level": 58.42362283522166, - "cpk": 19.20864243451263, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3002713918685913, - "mean_abs_diff": 0.04833799973130226, - "cosine_similarity": 0.9998859763145447, - "kl_divergence": 0.002065220995085909, - "sigma_level": 65.56426418444426, - "cpk": 21.59065094610386, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2973330020904541, - "mean_abs_diff": 0.043786294758319855, - "cosine_similarity": 0.9999096393585205, - "kl_divergence": 0.0006713194206523875, - "sigma_level": 71.27627178747183, - "cpk": 23.49868027534391, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.260500431060791, - "mean_abs_diff": 0.03889676183462143, - "cosine_similarity": 0.9999371767044067, - "kl_divergence": 0.0008542435718797348, - "sigma_level": 80.76395376693847, - "cpk": 26.65952989943824, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.27887630462646484, - "mean_abs_diff": 0.04313310235738754, - "cosine_similarity": 0.9999385476112366, - "kl_divergence": 0.0013764086829698126, - "sigma_level": 73.31553745426578, - "cpk": 24.174985269638448, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.234965980052948, - "mean_abs_diff": 0.037016309797763824, - "cosine_similarity": 0.9998894929885864, - "kl_divergence": 0.0002517016344552702, - "sigma_level": 85.95779722211104, - "cpk": 28.387445703078093, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2571530342102051, - "mean_abs_diff": 0.04071902483701706, - "cosine_similarity": 0.9998915791511536, - "kl_divergence": 0.002276788640036869, - "sigma_level": 76.79133459599869, - "cpk": 25.33653917694272, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.28516435623168945, - "mean_abs_diff": 0.04774056375026703, - "cosine_similarity": 0.9999310970306396, - "kl_divergence": 0.00042356634355521976, - "sigma_level": 68.00796327852372, - "cpk": 22.398759550639227, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.26332998275756836, - "mean_abs_diff": 0.0412241667509079, - "cosine_similarity": 0.9999069571495056, - "kl_divergence": 0.0015880368535440277, - "sigma_level": 76.05695528178228, - "cpk": 25.091036376668878, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.27721452713012695, - "mean_abs_diff": 0.049495816230773926, - "cosine_similarity": 0.9999117851257324, - "kl_divergence": 0.0010624357608136137, - "sigma_level": 65.57931661159489, - "cpk": 21.589280386902693, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.27029943466186523, - "mean_abs_diff": 0.05219271406531334, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.0010408474118303833, - "sigma_level": 63.40393435111078, - "cpk": 20.85887616568664, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3171241283416748, - "mean_abs_diff": 0.0380561500787735, - "cosine_similarity": 0.9999051690101624, - "kl_divergence": 0.0005325506114771998, - "sigma_level": 82.97525799995765, - "cpk": 27.395276094046604, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.24947690963745117, - "mean_abs_diff": 0.04606899619102478, - "cosine_similarity": 0.9999382495880127, - "kl_divergence": 0.0010556729962325755, - "sigma_level": 70.20693362744514, - "cpk": 23.13278096265952, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 1030, - "gpu_argmax": 1030, - "max_abs_diff": 0.2380300760269165, - "mean_abs_diff": 0.042963165789842606, - "cosine_similarity": 0.9999196529388428, - "kl_divergence": 0.00031761659666717314, - "sigma_level": 74.05351977421626, - "cpk": 24.419375454123664, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.25017356872558594, - "mean_abs_diff": 0.042943935841321945, - "cosine_similarity": 0.9999315142631531, - "kl_divergence": 0.0008225602750564301, - "sigma_level": 73.91521235873138, - "cpk": 24.373886608141206, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2465496063232422, - "mean_abs_diff": 0.04862823337316513, - "cosine_similarity": 0.9999127984046936, - "kl_divergence": 0.0005214034063651669, - "sigma_level": 67.83318248297935, - "cpk": 22.336176841974215, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.21641117334365845, - "mean_abs_diff": 0.035992056131362915, - "cosine_similarity": 0.9998767971992493, - "kl_divergence": 0.0010573586805331199, - "sigma_level": 87.34196263582032, - "cpk": 28.85201947678912, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.229217529296875, - "mean_abs_diff": 0.03840841352939606, - "cosine_similarity": 0.9999101161956787, - "kl_divergence": 0.001288439229203474, - "sigma_level": 82.85066673786027, - "cpk": 27.351708690182264, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.2576260566711426, - "mean_abs_diff": 0.043872181326150894, - "cosine_similarity": 0.9999276995658875, - "kl_divergence": 0.0013946707343927692, - "sigma_level": 73.02528098369798, - "cpk": 24.0747787971735, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.27596378326416016, - "mean_abs_diff": 0.05197622999548912, - "cosine_similarity": 0.999906599521637, - "kl_divergence": 0.0007940310091461293, - "sigma_level": 62.95701417717737, - "cpk": 20.71298237166727, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.21891295909881592, - "mean_abs_diff": 0.037558142095804214, - "cosine_similarity": 0.9999196529388428, - "kl_divergence": 0.00008131892636760107, - "sigma_level": 83.8883017292411, - "cpk": 27.70020984670349, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.21148943901062012, - "mean_abs_diff": 0.03717939928174019, - "cosine_similarity": 0.9999348521232605, - "kl_divergence": 0.00015387547222722972, - "sigma_level": 85.37753889289596, - "cpk": 28.194655830282713, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.2474832534790039, - "mean_abs_diff": 0.04211854934692383, - "cosine_similarity": 0.9999555349349976, - "kl_divergence": 0.0006737847648512362, - "sigma_level": 76.38969649351016, - "cpk": 25.19511356439018, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.21636962890625, - "mean_abs_diff": 0.035941481590270996, - "cosine_similarity": 0.9999298453330994, - "kl_divergence": 0.0004747428474169141, - "sigma_level": 88.5547413084973, - "cpk": 29.25301471912655, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2862367630004883, - "mean_abs_diff": 0.04804086312651634, - "cosine_similarity": 0.9999341368675232, - "kl_divergence": 0.0010998410243304767, - "sigma_level": 67.27443900835688, - "cpk": 22.15548615975949, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2140594720840454, - "mean_abs_diff": 0.03911084309220314, - "cosine_similarity": 0.999923586845398, - "kl_divergence": 0.0005905188974057968, - "sigma_level": 82.12074070221703, - "cpk": 27.105929283720684, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.26285481452941895, - "mean_abs_diff": 0.041882410645484924, - "cosine_similarity": 0.9999344944953918, - "kl_divergence": 0.0014377984084730992, - "sigma_level": 75.78547694290033, - "cpk": 24.99731910877624, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.26805317401885986, - "mean_abs_diff": 0.03873350843787193, - "cosine_similarity": 0.9999223947525024, - "kl_divergence": 0.0013953240937074677, - "sigma_level": 81.43229589631191, - "cpk": 26.881252088752714, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.21961069107055664, - "mean_abs_diff": 0.031359970569610596, - "cosine_similarity": 0.9999322295188904, - "kl_divergence": 0.0003661656672644013, - "sigma_level": 100.25205580134937, - "cpk": 33.155360140493684, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.24137163162231445, - "mean_abs_diff": 0.04070734232664108, - "cosine_similarity": 0.9999204277992249, - "kl_divergence": 0.0007682273957576004, - "sigma_level": 76.92578673738473, - "cpk": 25.38097521792283, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.22877216339111328, - "mean_abs_diff": 0.033897481858730316, - "cosine_similarity": 0.9999273419380188, - "kl_divergence": 0.0001278063653288519, - "sigma_level": 92.43480457949828, - "cpk": 30.550492600553692, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2641177177429199, - "mean_abs_diff": 0.05057620257139206, - "cosine_similarity": 0.9998951554298401, - "kl_divergence": 0.0000393485153266477, - "sigma_level": 64.30636835231786, - "cpk": 21.164425124737836, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.2049417495727539, - "mean_abs_diff": 0.034037526696920395, - "cosine_similarity": 0.9999337792396545, - "kl_divergence": 0.00027996516407924826, - "sigma_level": 93.34949438746834, - "cpk": 30.851715970209657, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2751140594482422, - "mean_abs_diff": 0.04576941952109337, - "cosine_similarity": 0.9999250173568726, - "kl_divergence": 0.00035061780104783284, - "sigma_level": 69.71321462919332, - "cpk": 22.971843762520475, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.22286415100097656, - "mean_abs_diff": 0.039300888776779175, - "cosine_similarity": 0.9999452829360962, - "kl_divergence": 0.0003919376483464612, - "sigma_level": 79.54308945859913, - "cpk": 26.253853643551892, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2212667465209961, - "mean_abs_diff": 0.03471957892179489, - "cosine_similarity": 0.9999411702156067, - "kl_divergence": 0.00031067307671760625, - "sigma_level": 91.65536909076584, - "cpk": 30.286603378525857, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.24101543426513672, - "mean_abs_diff": 0.03968901187181473, - "cosine_similarity": 0.9999310970306396, - "kl_divergence": 0.0009727681612350478, - "sigma_level": 79.04490093748886, - "cpk": 26.086865811520088, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.2663121223449707, - "mean_abs_diff": 0.041511133313179016, - "cosine_similarity": 0.9999287128448486, - "kl_divergence": 0.0007963946022263226, - "sigma_level": 76.45829411292644, - "cpk": 25.221608834323817, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2315669059753418, - "mean_abs_diff": 0.034654080867767334, - "cosine_similarity": 0.9999446272850037, - "kl_divergence": 0.0001058477485922799, - "sigma_level": 91.1050135078264, - "cpk": 30.105241127145526, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.3170022964477539, - "mean_abs_diff": 0.055937547236680984, - "cosine_similarity": 0.9998112916946411, - "kl_divergence": 0.0011604884668601218, - "sigma_level": 57.829405751003826, - "cpk": 19.00689899067911, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 16, + "gpu_argmax": 16, + "max_abs_diff": 0.17002153396606445, + "mean_abs_diff": 0.028188295662403107, + "cosine_similarity": 0.9999852776527405, + "kl_divergence": 0.0005291031695536818, + "sigma_level": 113.51094745441767, + "cpk": 37.57034247249217, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.31151044368743896, + "mean_abs_diff": 0.04776180535554886, + "cosine_similarity": 0.999913215637207, + "kl_divergence": 1.8919660545105944e-05, + "sigma_level": 65.37697481827193, + "cpk": 21.532114743923568, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.2960681915283203, + "mean_abs_diff": 0.05043606460094452, + "cosine_similarity": 0.9999374151229858, + "kl_divergence": 5.5291439027043675e-06, + "sigma_level": 61.84778951581121, + "cpk": 20.355983246316647, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.2933235168457031, + "mean_abs_diff": 0.04766261205077171, + "cosine_similarity": 0.9999246001243591, + "kl_divergence": 0.0001060628215343356, + "sigma_level": 66.19522509167993, + "cpk": 21.80215525279679, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.3401813507080078, + "mean_abs_diff": 0.05537569522857666, + "cosine_similarity": 0.9999253749847412, + "kl_divergence": 3.1806382760983147e-06, + "sigma_level": 58.14417981939996, + "cpk": 19.113078741383767, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.29981279373168945, + "mean_abs_diff": 0.04475972056388855, + "cosine_similarity": 0.9999157786369324, + "kl_divergence": 6.075290117685989e-05, + "sigma_level": 70.78162520727061, + "cpk": 23.32986125531226, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.36730432510375977, + "mean_abs_diff": 0.05289534851908684, + "cosine_similarity": 0.9999330639839172, + "kl_divergence": 3.942939467082102e-06, + "sigma_level": 59.221911860577734, + "cpk": 19.47959031453992, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.37870264053344727, + "mean_abs_diff": 0.06961360573768616, + "cosine_similarity": 0.9999417066574097, + "kl_divergence": 5.816292943734862e-06, + "sigma_level": 47.307308057754874, + "cpk": 15.494666661614641, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2391514778137207, + "mean_abs_diff": 0.03515106812119484, + "cosine_similarity": 0.9999060034751892, + "kl_divergence": 0.00020459017940060442, + "sigma_level": 89.95740355697005, + "cpk": 29.72229295062028, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.2996964454650879, + "mean_abs_diff": 0.04814895987510681, + "cosine_similarity": 0.9999247193336487, + "kl_divergence": 0.0008225811496027273, + "sigma_level": 66.99980088661925, + "cpk": 22.064436068495585, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.2887105941772461, + "mean_abs_diff": 0.05602011829614639, + "cosine_similarity": 0.9999487400054932, + "kl_divergence": 0.0017507166523780418, + "sigma_level": 58.404129958133645, + "cpk": 19.19539279694137, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.23575425148010254, + "mean_abs_diff": 0.03749267756938934, + "cosine_similarity": 0.9999302625656128, + "kl_divergence": 0.0009205649545698543, + "sigma_level": 83.41319884576228, + "cpk": 27.5437842678078, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.23073053359985352, + "mean_abs_diff": 0.036286987364292145, + "cosine_similarity": 0.9999309778213501, + "kl_divergence": 0.0002103350091946032, + "sigma_level": 87.46254652146891, + "cpk": 28.889702813783522, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.22650158405303955, + "mean_abs_diff": 0.03817339614033699, + "cosine_similarity": 0.9999290108680725, + "kl_divergence": 0.0006199522697201657, + "sigma_level": 82.87208180962669, + "cpk": 27.360401535884456, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.2506117820739746, + "mean_abs_diff": 0.04409490525722504, + "cosine_similarity": 0.9999329447746277, + "kl_divergence": 0.001207147840417756, + "sigma_level": 72.75087140793437, + "cpk": 23.982961904135337, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2462446689605713, + "mean_abs_diff": 0.03615713492035866, + "cosine_similarity": 0.9999027252197266, + "kl_divergence": 0.0007957232607366745, + "sigma_level": 86.4713156254837, + "cpk": 28.563225623010272, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.30622434616088867, + "mean_abs_diff": 0.04285147041082382, + "cosine_similarity": 0.99992436170578, + "kl_divergence": 0.001562816857886753, + "sigma_level": 73.57721644186985, + "cpk": 24.262997821184154, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.26041460037231445, + "mean_abs_diff": 0.041633863002061844, + "cosine_similarity": 0.9999330043792725, + "kl_divergence": 3.86192979878687e-05, + "sigma_level": 76.17994851863548, + "cpk": 25.129010711201076, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.2938199043273926, + "mean_abs_diff": 0.04622060805559158, + "cosine_similarity": 0.9999043345451355, + "kl_divergence": 0.0004776241426479791, + "sigma_level": 69.31179649604657, + "cpk": 22.836962717059464, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.33150339126586914, + "mean_abs_diff": 0.05491165816783905, + "cosine_similarity": 0.9998782873153687, + "kl_divergence": 0.004729332624206632, + "sigma_level": 59.30963249897647, + "cpk": 19.498478310838493, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.2719573974609375, + "mean_abs_diff": 0.0452549010515213, + "cosine_similarity": 0.9998787045478821, + "kl_divergence": 0.001985488312073216, + "sigma_level": 69.17673829662412, + "cpk": 22.798030561651316, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.5426011085510254, + "mean_abs_diff": 0.08933572471141815, + "cosine_similarity": 0.9997243881225586, + "kl_divergence": 0.0025612063383940826, + "sigma_level": 35.23386750744257, + "cpk": 11.482318911800565, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.6198992729187012, + "mean_abs_diff": 0.09157062321901321, + "cosine_similarity": 0.9995829463005066, + "kl_divergence": 0.010837422875741744, + "sigma_level": 34.05332167588209, + "cpk": 11.091250234582526, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.6332998275756836, + "mean_abs_diff": 0.0804496556520462, + "cosine_similarity": 0.9996832013130188, + "kl_divergence": 0.002525993250052904, + "sigma_level": 38.54398452241279, + "cpk": 12.589590650613772, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.35448646545410156, + "mean_abs_diff": 0.05539744719862938, + "cosine_similarity": 0.9998620748519897, + "kl_divergence": 0.002686264662680092, + "sigma_level": 56.62337561525491, + "cpk": 18.613059333347113, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3623623847961426, + "mean_abs_diff": 0.05665629357099533, + "cosine_similarity": 0.9997888207435608, + "kl_divergence": 6.107804999826922e-05, + "sigma_level": 57.59732769966832, + "cpk": 18.927171640968005, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.26375818252563477, + "mean_abs_diff": 0.04152446612715721, + "cosine_similarity": 0.9999150633811951, + "kl_divergence": 0.00028931538420952634, + "sigma_level": 75.91556809492718, + "cpk": 25.042493245318916, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.298952579498291, + "mean_abs_diff": 0.05040597915649414, + "cosine_similarity": 0.9997300505638123, + "kl_divergence": 0.00029820851684663813, + "sigma_level": 63.07446362066332, + "cpk": 20.759877032006923, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.335299015045166, + "mean_abs_diff": 0.06050831079483032, + "cosine_similarity": 0.9998612999916077, + "kl_divergence": 0.002278695086270758, + "sigma_level": 52.97888535424999, + "cpk": 17.392489879701778, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3436293601989746, + "mean_abs_diff": 0.05135822668671608, + "cosine_similarity": 0.9998526573181152, + "kl_divergence": 0.001024066884138274, + "sigma_level": 61.078798535758686, + "cpk": 20.098191280173584, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2161470353603363, + "mean_abs_diff": 0.03684602305293083, + "cosine_similarity": 0.9998604655265808, + "kl_divergence": 3.7301007973426357e-05, + "sigma_level": 84.97813534510328, + "cpk": 28.0651195872077, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3007354736328125, + "mean_abs_diff": 0.05120687186717987, + "cosine_similarity": 0.9999154806137085, + "kl_divergence": 0.0007108443470111474, + "sigma_level": 63.124975763940014, + "cpk": 20.772289209183093, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.28400611877441406, + "mean_abs_diff": 0.048652347177267075, + "cosine_similarity": 0.999927282333374, + "kl_divergence": 0.0015689714082204687, + "sigma_level": 65.72234032440733, + "cpk": 21.640984598238646, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 1075, + "gpu_argmax": 1075, + "max_abs_diff": 0.2669934034347534, + "mean_abs_diff": 0.04793161153793335, + "cosine_similarity": 0.9999324083328247, + "kl_divergence": 0.0025805480581009115, + "sigma_level": 67.08375790360458, + "cpk": 22.09329991583983, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.23727846145629883, + "mean_abs_diff": 0.0352792851626873, + "cosine_similarity": 0.9998976588249207, + "kl_divergence": 3.295202382643496e-05, + "sigma_level": 89.6666882397245, + "cpk": 29.625281357907912, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.33716487884521484, + "mean_abs_diff": 0.05362644046545029, + "cosine_similarity": 0.9999046921730042, + "kl_divergence": 0.004216564679063551, + "sigma_level": 58.67804631412172, + "cpk": 19.297124208266144, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.3685007095336914, + "mean_abs_diff": 0.06397877633571625, + "cosine_similarity": 0.9996156692504883, + "kl_divergence": 0.005635637283317798, + "sigma_level": 50.13709882194314, + "cpk": 16.445057088010145, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.2645277976989746, + "mean_abs_diff": 0.034263480454683304, + "cosine_similarity": 0.9998966455459595, + "kl_divergence": 0.00022884919989930267, + "sigma_level": 91.22137614961048, + "cpk": 30.146661896640868, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.34772682189941406, + "mean_abs_diff": 0.0546145886182785, + "cosine_similarity": 0.9999197125434875, + "kl_divergence": 0.0028962852285730335, + "sigma_level": 58.42362283522166, + "cpk": 19.20864243451263, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3002713918685913, + "mean_abs_diff": 0.04833799973130226, + "cosine_similarity": 0.9998859763145447, + "kl_divergence": 0.002065220995085909, + "sigma_level": 65.56426418444426, + "cpk": 21.59065094610386, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2973330020904541, + "mean_abs_diff": 0.043786294758319855, + "cosine_similarity": 0.9999096393585205, + "kl_divergence": 0.0006713194206523875, + "sigma_level": 71.27627178747183, + "cpk": 23.49868027534391, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.260500431060791, + "mean_abs_diff": 0.03889676183462143, + "cosine_similarity": 0.9999371767044067, + "kl_divergence": 0.0008542435718797348, + "sigma_level": 80.76395376693847, + "cpk": 26.65952989943824, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.27887630462646484, + "mean_abs_diff": 0.04313310235738754, + "cosine_similarity": 0.9999385476112366, + "kl_divergence": 0.0013764086829698126, + "sigma_level": 73.31553745426578, + "cpk": 24.174985269638448, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.234965980052948, + "mean_abs_diff": 0.037016309797763824, + "cosine_similarity": 0.9998894929885864, + "kl_divergence": 0.0002517016344552702, + "sigma_level": 85.95779722211104, + "cpk": 28.387445703078093, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2571530342102051, + "mean_abs_diff": 0.04071902483701706, + "cosine_similarity": 0.9998915791511536, + "kl_divergence": 0.002276788640036869, + "sigma_level": 76.79133459599869, + "cpk": 25.33653917694272, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.28516435623168945, + "mean_abs_diff": 0.04774056375026703, + "cosine_similarity": 0.9999310970306396, + "kl_divergence": 0.00042356634355521976, + "sigma_level": 68.00796327852372, + "cpk": 22.398759550639227, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.26332998275756836, + "mean_abs_diff": 0.0412241667509079, + "cosine_similarity": 0.9999069571495056, + "kl_divergence": 0.0015880368535440277, + "sigma_level": 76.05695528178228, + "cpk": 25.091036376668878, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.27721452713012695, + "mean_abs_diff": 0.049495816230773926, + "cosine_similarity": 0.9999117851257324, + "kl_divergence": 0.0010624357608136137, + "sigma_level": 65.57931661159489, + "cpk": 21.589280386902693, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.27029943466186523, + "mean_abs_diff": 0.05219271406531334, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 0.0010408474118303833, + "sigma_level": 63.40393435111078, + "cpk": 20.85887616568664, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3171241283416748, + "mean_abs_diff": 0.0380561500787735, + "cosine_similarity": 0.9999051690101624, + "kl_divergence": 0.0005325506114771998, + "sigma_level": 82.97525799995765, + "cpk": 27.395276094046604, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.24947690963745117, + "mean_abs_diff": 0.04606899619102478, + "cosine_similarity": 0.9999382495880127, + "kl_divergence": 0.0010556729962325755, + "sigma_level": 70.20693362744514, + "cpk": 23.13278096265952, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 1030, + "gpu_argmax": 1030, + "max_abs_diff": 0.2380300760269165, + "mean_abs_diff": 0.042963165789842606, + "cosine_similarity": 0.9999196529388428, + "kl_divergence": 0.00031761659666717314, + "sigma_level": 74.05351977421626, + "cpk": 24.419375454123664, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.25017356872558594, + "mean_abs_diff": 0.042943935841321945, + "cosine_similarity": 0.9999315142631531, + "kl_divergence": 0.0008225602750564301, + "sigma_level": 73.91521235873138, + "cpk": 24.373886608141206, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2465496063232422, + "mean_abs_diff": 0.04862823337316513, + "cosine_similarity": 0.9999127984046936, + "kl_divergence": 0.0005214034063651669, + "sigma_level": 67.83318248297935, + "cpk": 22.336176841974215, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.21641117334365845, + "mean_abs_diff": 0.035992056131362915, + "cosine_similarity": 0.9998767971992493, + "kl_divergence": 0.0010573586805331199, + "sigma_level": 87.34196263582032, + "cpk": 28.85201947678912, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.229217529296875, + "mean_abs_diff": 0.03840841352939606, + "cosine_similarity": 0.9999101161956787, + "kl_divergence": 0.001288439229203474, + "sigma_level": 82.85066673786027, + "cpk": 27.351708690182264, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.2576260566711426, + "mean_abs_diff": 0.043872181326150894, + "cosine_similarity": 0.9999276995658875, + "kl_divergence": 0.0013946707343927692, + "sigma_level": 73.02528098369798, + "cpk": 24.0747787971735, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.27596378326416016, + "mean_abs_diff": 0.05197622999548912, + "cosine_similarity": 0.999906599521637, + "kl_divergence": 0.0007940310091461293, + "sigma_level": 62.95701417717737, + "cpk": 20.71298237166727, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.21891295909881592, + "mean_abs_diff": 0.037558142095804214, + "cosine_similarity": 0.9999196529388428, + "kl_divergence": 8.131892636760107e-05, + "sigma_level": 83.8883017292411, + "cpk": 27.70020984670349, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.21148943901062012, + "mean_abs_diff": 0.03717939928174019, + "cosine_similarity": 0.9999348521232605, + "kl_divergence": 0.00015387547222722972, + "sigma_level": 85.37753889289596, + "cpk": 28.194655830282713, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.2474832534790039, + "mean_abs_diff": 0.04211854934692383, + "cosine_similarity": 0.9999555349349976, + "kl_divergence": 0.0006737847648512362, + "sigma_level": 76.38969649351016, + "cpk": 25.19511356439018, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.21636962890625, + "mean_abs_diff": 0.035941481590270996, + "cosine_similarity": 0.9999298453330994, + "kl_divergence": 0.0004747428474169141, + "sigma_level": 88.5547413084973, + "cpk": 29.25301471912655, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2862367630004883, + "mean_abs_diff": 0.04804086312651634, + "cosine_similarity": 0.9999341368675232, + "kl_divergence": 0.0010998410243304767, + "sigma_level": 67.27443900835688, + "cpk": 22.15548615975949, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2140594720840454, + "mean_abs_diff": 0.03911084309220314, + "cosine_similarity": 0.999923586845398, + "kl_divergence": 0.0005905188974057968, + "sigma_level": 82.12074070221703, + "cpk": 27.105929283720684, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.26285481452941895, + "mean_abs_diff": 0.041882410645484924, + "cosine_similarity": 0.9999344944953918, + "kl_divergence": 0.0014377984084730992, + "sigma_level": 75.78547694290033, + "cpk": 24.99731910877624, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.26805317401885986, + "mean_abs_diff": 0.03873350843787193, + "cosine_similarity": 0.9999223947525024, + "kl_divergence": 0.0013953240937074677, + "sigma_level": 81.43229589631191, + "cpk": 26.881252088752714, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.21961069107055664, + "mean_abs_diff": 0.031359970569610596, + "cosine_similarity": 0.9999322295188904, + "kl_divergence": 0.0003661656672644013, + "sigma_level": 100.25205580134937, + "cpk": 33.155360140493684, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.24137163162231445, + "mean_abs_diff": 0.04070734232664108, + "cosine_similarity": 0.9999204277992249, + "kl_divergence": 0.0007682273957576004, + "sigma_level": 76.92578673738473, + "cpk": 25.38097521792283, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.22877216339111328, + "mean_abs_diff": 0.033897481858730316, + "cosine_similarity": 0.9999273419380188, + "kl_divergence": 0.0001278063653288519, + "sigma_level": 92.43480457949828, + "cpk": 30.550492600553692, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2641177177429199, + "mean_abs_diff": 0.05057620257139206, + "cosine_similarity": 0.9998951554298401, + "kl_divergence": 3.93485153266477e-05, + "sigma_level": 64.30636835231786, + "cpk": 21.164425124737836, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.2049417495727539, + "mean_abs_diff": 0.034037526696920395, + "cosine_similarity": 0.9999337792396545, + "kl_divergence": 0.00027996516407924826, + "sigma_level": 93.34949438746834, + "cpk": 30.851715970209657, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2751140594482422, + "mean_abs_diff": 0.04576941952109337, + "cosine_similarity": 0.9999250173568726, + "kl_divergence": 0.00035061780104783284, + "sigma_level": 69.71321462919332, + "cpk": 22.971843762520475, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.22286415100097656, + "mean_abs_diff": 0.039300888776779175, + "cosine_similarity": 0.9999452829360962, + "kl_divergence": 0.0003919376483464612, + "sigma_level": 79.54308945859913, + "cpk": 26.253853643551892, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2212667465209961, + "mean_abs_diff": 0.03471957892179489, + "cosine_similarity": 0.9999411702156067, + "kl_divergence": 0.00031067307671760625, + "sigma_level": 91.65536909076584, + "cpk": 30.286603378525857, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.24101543426513672, + "mean_abs_diff": 0.03968901187181473, + "cosine_similarity": 0.9999310970306396, + "kl_divergence": 0.0009727681612350478, + "sigma_level": 79.04490093748886, + "cpk": 26.086865811520088, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.2663121223449707, + "mean_abs_diff": 0.041511133313179016, + "cosine_similarity": 0.9999287128448486, + "kl_divergence": 0.0007963946022263226, + "sigma_level": 76.45829411292644, + "cpk": 25.221608834323817, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2315669059753418, + "mean_abs_diff": 0.034654080867767334, + "cosine_similarity": 0.9999446272850037, + "kl_divergence": 0.0001058477485922799, + "sigma_level": 91.1050135078264, + "cpk": 30.105241127145526, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.3170022964477539, + "mean_abs_diff": 0.055937547236680984, + "cosine_similarity": 0.9998112916946411, + "kl_divergence": 0.0011604884668601218, + "sigma_level": 57.829405751003826, + "cpk": 19.00689899067911, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json index 9ddff2dde2..98aa405f04 100644 --- a/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "8cdb7c1a668127db", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md names the binary but carries no timestamp; the date is the git author date of the commit that ADDED this record \u2014 an upper bound on when it was measured, stated as such and not as a measurement", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9997819066047668, + "min_cosine_position": 24, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.22080087661743164, - "mean_abs_diff": 0.029157055541872978, - "cosine_similarity": 0.9999111890792847, - "kl_divergence": 0.00032579414447661754, - "sigma_level": 108.38003583659224, - "cpk": 35.863341718487604, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.1777787208557129, - "mean_abs_diff": 0.03252752497792244, - "cosine_similarity": 0.9999105930328369, - "kl_divergence": 3.7201543263529397e-6, - "sigma_level": 99.81459433877987, - "cpk": 33.000971303716966, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.17074143886566162, - "mean_abs_diff": 0.024813607335090637, - "cosine_similarity": 0.9999530911445618, - "kl_divergence": 6.065099658800252e-6, - "sigma_level": 126.29212369865193, - "cpk": 41.83622763563623, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.17793822288513184, - "mean_abs_diff": 0.029602359980344772, - "cosine_similarity": 0.9999439120292664, - "kl_divergence": 0.000021107606354716864, - "sigma_level": 106.68227318083109, - "cpk": 35.29758713909199, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.21715784072875977, - "mean_abs_diff": 0.033003006130456924, - "cosine_similarity": 0.9999298453330994, - "kl_divergence": 4.5147641864858906e-7, - "sigma_level": 97.43239515014821, - "cpk": 32.20950155551228, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.16768229007720947, - "mean_abs_diff": 0.026923540979623795, - "cosine_similarity": 0.9999296069145203, - "kl_divergence": 0.000024167411428152043, - "sigma_level": 115.93981600656502, - "cpk": 38.38647946990311, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.1763458251953125, - "mean_abs_diff": 0.027678435668349266, - "cosine_similarity": 0.9999611973762512, - "kl_divergence": 1.2421132461965687e-6, - "sigma_level": 114.27627514935318, - "cpk": 37.8285093389394, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.2084965705871582, - "mean_abs_diff": 0.03252236545085907, - "cosine_similarity": 0.9999578595161438, - "kl_divergence": 1.444100421606142e-6, - "sigma_level": 98.27470173458114, - "cpk": 32.4918900978282, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2644195556640625, - "mean_abs_diff": 0.02975749969482422, - "cosine_similarity": 0.9999121427536011, - "kl_divergence": 0.0002420612186183089, - "sigma_level": 105.07405871764146, - "cpk": 34.764124466695144, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.21603012084960938, - "mean_abs_diff": 0.03684907406568527, - "cosine_similarity": 0.9999308586120605, - "kl_divergence": 0.0006474370152407752, - "sigma_level": 86.82147331522204, - "cpk": 28.67388353001697, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.2456355094909668, - "mean_abs_diff": 0.04065067693591118, - "cosine_similarity": 0.9999247193336487, - "kl_divergence": 0.001367731303832384, - "sigma_level": 78.56056947133877, - "cpk": 25.920728129656204, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.18509769439697266, - "mean_abs_diff": 0.03018302470445633, - "cosine_similarity": 0.9999426007270813, - "kl_divergence": 0.0005161167428984038, - "sigma_level": 104.15563778996108, - "cpk": 34.45656824761013, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.17779541015625, - "mean_abs_diff": 0.031838010996580124, - "cosine_similarity": 0.999921977519989, - "kl_divergence": 0.00017258435469219227, - "sigma_level": 99.16622380548698, - "cpk": 32.792303324828296, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.1694812774658203, - "mean_abs_diff": 0.026445960626006126, - "cosine_similarity": 0.9999368786811829, - "kl_divergence": 0.0005906903382915679, - "sigma_level": 118.52313191771943, - "cpk": 39.24650579924256, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.2208271026611328, - "mean_abs_diff": 0.0328725203871727, - "cosine_similarity": 0.9999487400054932, - "kl_divergence": 0.0007866923538006692, - "sigma_level": 96.84132305510252, - "cpk": 32.01515615449671, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18550491333007812, - "mean_abs_diff": 0.026621753349900246, - "cosine_similarity": 0.999907374382019, - "kl_divergence": 0.0002935938926612731, - "sigma_level": 117.17077206714823, - "cpk": 38.796983072900325, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.22534465789794922, - "mean_abs_diff": 0.035026710480451584, - "cosine_similarity": 0.9999264478683472, - "kl_divergence": 0.00047285014688066114, - "sigma_level": 90.85841261803856, - "cpk": 30.020931596555734, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.14307236671447754, - "mean_abs_diff": 0.02470676600933075, - "cosine_similarity": 0.999955952167511, - "kl_divergence": 0.00003484863188887505, - "sigma_level": 128.68804300700253, - "cpk": 42.631058888436456, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.19958877563476562, - "mean_abs_diff": 0.028368674218654633, - "cosine_similarity": 0.9999251365661621, - "kl_divergence": 0.00007519966075083051, - "sigma_level": 112.12267875571543, - "cpk": 37.1091619397265, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 5942, - "gpu_argmax": 5942, - "max_abs_diff": 0.1851489543914795, - "mean_abs_diff": 0.03547385707497597, - "cosine_similarity": 0.9999073147773743, - "kl_divergence": 0.0005868901702431452, - "sigma_level": 92.18467398700828, - "cpk": 30.455712499876228, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.2207651138305664, - "mean_abs_diff": 0.03258340433239937, - "cosine_similarity": 0.9999098777770996, - "kl_divergence": 0.0009104736533284513, - "sigma_level": 96.75204615803823, - "cpk": 31.98797279934988, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.2389364242553711, - "mean_abs_diff": 0.038926344364881516, - "cosine_similarity": 0.9999305009841919, - "kl_divergence": 0.000779499423692475, - "sigma_level": 80.85147160964138, - "cpk": 26.688219517690097, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2731236219406128, - "mean_abs_diff": 0.04174664989113808, - "cosine_similarity": 0.9998652935028076, - "kl_divergence": 0.0016339679460959167, - "sigma_level": 74.70249389609015, - "cpk": 24.64094972714047, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.24706506729125977, - "mean_abs_diff": 0.04125778004527092, - "cosine_similarity": 0.9998652935028076, - "kl_divergence": 0.00012628074726192903, - "sigma_level": 77.1027654306697, - "cpk": 25.43583106547151, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.23851919174194336, - "mean_abs_diff": 0.043194543570280075, - "cosine_similarity": 0.9997819066047668, - "kl_divergence": 0.0007928120095410873, - "sigma_level": 75.55819186163643, - "cpk": 24.914088819673893, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1764129400253296, - "mean_abs_diff": 0.026423683390021324, - "cosine_similarity": 0.9998510479927063, - "kl_divergence": 0.00003542603986894582, - "sigma_level": 118.8965940005014, - "cpk": 39.37039083716537, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19931602478027344, - "mean_abs_diff": 0.03260093927383423, - "cosine_similarity": 0.9998522400856018, - "kl_divergence": 0.00016412521494609694, - "sigma_level": 98.43831627071472, - "cpk": 32.54534029265824, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.16838788986206055, - "mean_abs_diff": 0.025229420512914658, - "cosine_similarity": 0.9998432993888855, - "kl_divergence": 0.00022452636227061513, - "sigma_level": 124.04995771005096, - "cpk": 41.08917685771064, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.18775105476379395, - "mean_abs_diff": 0.028898026794195175, - "cosine_similarity": 0.9998990893363953, - "kl_divergence": 0.000461885527735682, - "sigma_level": 108.11494274168895, - "cpk": 35.77795520454613, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.1990138292312622, - "mean_abs_diff": 0.02971040830016136, - "cosine_similarity": 0.9999052882194519, - "kl_divergence": 0.00019171666737919304, - "sigma_level": 107.355287869335, - "cpk": 35.51929850346339, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1763872504234314, - "mean_abs_diff": 0.02600252814590931, - "cosine_similarity": 0.9998703598976135, - "kl_divergence": 5.419791774745481e-6, - "sigma_level": 121.23067932375403, - "cpk": 40.14753442864604, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19364482164382935, - "mean_abs_diff": 0.027832137420773506, - "cosine_similarity": 0.9999036192893982, - "kl_divergence": 0.00015532038116852645, - "sigma_level": 112.43699543023251, - "cpk": 37.21821831774474, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.1895672082901001, - "mean_abs_diff": 0.028663579374551773, - "cosine_similarity": 0.9999141097068787, - "kl_divergence": 0.00048805987877351045, - "sigma_level": 110.13832391530234, - "cpk": 36.44969475595692, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.1705029010772705, - "mean_abs_diff": 0.02920815348625183, - "cosine_similarity": 0.9999076724052429, - "kl_divergence": 0.0006465286747198685, - "sigma_level": 107.00006736446687, - "cpk": 35.40624958893724, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.17264866828918457, - "mean_abs_diff": 0.027726825326681137, - "cosine_similarity": 0.9998999238014221, - "kl_divergence": 9.778256838183266e-6, - "sigma_level": 113.50747960523304, - "cpk": 37.5735596967205, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.20781350135803223, - "mean_abs_diff": 0.0317465104162693, - "cosine_similarity": 0.9999275803565979, - "kl_divergence": 0.0007944463203620559, - "sigma_level": 98.32937454807063, - "cpk": 32.516323639913914, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.16794657707214355, - "mean_abs_diff": 0.025144487619400024, - "cosine_similarity": 0.9998830556869507, - "kl_divergence": 0.0014841781022014834, - "sigma_level": 124.04210451408106, - "cpk": 41.08745357459047, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.14394879341125488, - "mean_abs_diff": 0.024181175976991653, - "cosine_similarity": 0.99992835521698, - "kl_divergence": 0.00004590317076718935, - "sigma_level": 131.6329585448339, - "cpk": 43.61239953703258, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.20485520362854004, - "mean_abs_diff": 0.032248660922050476, - "cosine_similarity": 0.9999155402183533, - "kl_divergence": 0.0010132197408340808, - "sigma_level": 97.76611926328172, - "cpk": 32.32597088527838, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.17113685607910156, - "mean_abs_diff": 0.027429627254605293, - "cosine_similarity": 0.9999209046363831, - "kl_divergence": 0.00020635141816486075, - "sigma_level": 114.89910887906412, - "cpk": 38.03706631565145, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.20895957946777344, - "mean_abs_diff": 0.03152859956026077, - "cosine_similarity": 0.9999305605888367, - "kl_divergence": 0.0012345569716387572, - "sigma_level": 100.7526951257938, - "cpk": 33.31951576032809, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2068023681640625, - "mean_abs_diff": 0.03304615244269371, - "cosine_similarity": 0.9999384880065918, - "kl_divergence": 0.0008537524363857488, - "sigma_level": 95.37599649244608, - "cpk": 31.52934802086009, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.19442176818847656, - "mean_abs_diff": 0.03274036571383476, - "cosine_similarity": 0.999926745891571, - "kl_divergence": 0.00022415574672985837, - "sigma_level": 96.70010563019095, - "cpk": 31.969535474822074, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.1620619297027588, - "mean_abs_diff": 0.02630770020186901, - "cosine_similarity": 0.9999019503593445, - "kl_divergence": 0.00015874297246952666, - "sigma_level": 119.79814893017625, - "cpk": 39.670081827825925, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2568702697753906, - "mean_abs_diff": 0.041561197489500046, - "cosine_similarity": 0.9999032020568848, - "kl_divergence": 0.0006645838048104513, - "sigma_level": 75.64519761529297, - "cpk": 24.953073788662536, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.23499298095703125, - "mean_abs_diff": 0.03136107325553894, - "cosine_similarity": 0.9999419450759888, - "kl_divergence": 0.0010564909150774027, - "sigma_level": 100.73015249089858, - "cpk": 33.31346702269047, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.19125032424926758, - "mean_abs_diff": 0.03027286008000374, - "cosine_similarity": 0.9999489188194275, - "kl_divergence": 0.0007324182535427381, - "sigma_level": 103.65502957340333, - "cpk": 34.290182007229085, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.17596006393432617, - "mean_abs_diff": 0.028325680643320084, - "cosine_similarity": 0.9999551177024841, - "kl_divergence": 0.0004250593527600838, - "sigma_level": 111.41886548737655, - "cpk": 36.87662056233914, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1672215461730957, - "mean_abs_diff": 0.02670413628220558, - "cosine_similarity": 0.9999512434005737, - "kl_divergence": 0.0003977476078045197, - "sigma_level": 118.5229880052167, - "cpk": 39.24390816638344, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.18988656997680664, - "mean_abs_diff": 0.029908394441008568, - "cosine_similarity": 0.999937117099762, - "kl_divergence": 0.00020613579089726636, - "sigma_level": 105.60082770042673, - "cpk": 34.9370799661288, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.18568384647369385, - "mean_abs_diff": 0.02834681235253811, - "cosine_similarity": 0.9999350309371948, - "kl_divergence": 0.0006665192974476394, - "sigma_level": 110.43670259979491, - "cpk": 36.55135682614585, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.20452356338500977, - "mean_abs_diff": 0.0283601526170969, - "cosine_similarity": 0.9999455809593201, - "kl_divergence": 0.00007353835996759626, - "sigma_level": 110.17008996001424, - "cpk": 36.46299327291264, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.17694008350372314, - "mean_abs_diff": 0.029460342600941658, - "cosine_similarity": 0.999952495098114, - "kl_divergence": 0.000354042526327484, - "sigma_level": 108.24661766438967, - "cpk": 35.81645735131439, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.17319059371948242, - "mean_abs_diff": 0.025698617100715637, - "cosine_similarity": 0.9999451041221619, - "kl_divergence": 0.0012082822780696116, - "sigma_level": 121.84608568951509, - "cpk": 40.35442223805871, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.18508267402648926, - "mean_abs_diff": 0.026767021045088768, - "cosine_similarity": 0.9999246597290039, - "kl_divergence": 8.510791162766133e-6, - "sigma_level": 117.67908059360465, - "cpk": 38.96386699563357, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.23066043853759766, - "mean_abs_diff": 0.036110661923885345, - "cosine_similarity": 0.9999273419380188, - "kl_divergence": 0.0004994907989914782, - "sigma_level": 88.93142368056236, - "cpk": 29.3761935122759, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.18856239318847656, - "mean_abs_diff": 0.030129026621580124, - "cosine_similarity": 0.9999247789382935, - "kl_divergence": 0.0008435059720457446, - "sigma_level": 104.45399344155254, - "cpk": 34.55573971808994, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.16795825958251953, - "mean_abs_diff": 0.0249911118298769, - "cosine_similarity": 0.9999393820762634, - "kl_divergence": 0.0001998253593928442, - "sigma_level": 124.82156616889559, - "cpk": 41.34723624638959, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.17892026901245117, - "mean_abs_diff": 0.028010983020067215, - "cosine_similarity": 0.9999496936798096, - "kl_divergence": 0.000045661537655344266, - "sigma_level": 112.21705530380771, - "cpk": 37.143742598712834, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.23161697387695312, - "mean_abs_diff": 0.03567603975534439, - "cosine_similarity": 0.9999344348907471, - "kl_divergence": 0.0002818625074985845, - "sigma_level": 90.15571250675596, - "cpk": 29.78387093695512, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.17027735710144043, - "mean_abs_diff": 0.026973217725753784, - "cosine_similarity": 0.9999549388885498, - "kl_divergence": 0.0005629856415008736, - "sigma_level": 116.05363658933409, - "cpk": 38.423683862478896, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.1832141876220703, - "mean_abs_diff": 0.02868635207414627, - "cosine_similarity": 0.999939501285553, - "kl_divergence": 0.0006119540337820587, - "sigma_level": 109.6258088123588, - "cpk": 36.27987255845257, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.2024369239807129, - "mean_abs_diff": 0.0438840389251709, - "cosine_similarity": 0.9999357461929321, - "kl_divergence": 0.00047113813301219134, - "sigma_level": 75.99682576801438, - "cpk": 25.054354617655378, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.18555545806884766, - "mean_abs_diff": 0.025885246694087982, - "cosine_similarity": 0.9999472498893738, - "kl_divergence": 0.000672849308967897, - "sigma_level": 121.69151806777218, - "cpk": 40.301338108777195, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.16499638557434082, - "mean_abs_diff": 0.02714230865240097, - "cosine_similarity": 0.9999595284461975, - "kl_divergence": 0.000296325939274267, - "sigma_level": 114.99287429648993, - "cpk": 38.070860424914805, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 9079, - "gpu_argmax": 3589, - "max_abs_diff": 0.17669677734375, - "mean_abs_diff": 0.02647567354142666, - "cosine_similarity": 0.9999456405639648, - "kl_divergence": 0.00043440203458147177, - "sigma_level": 119.37552964782644, - "cpk": 39.52846425329292, - "verdict": "WarnArgmax" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.15739870071411133, - "mean_abs_diff": 0.0278735663741827, - "cosine_similarity": 0.999921977519989, - "kl_divergence": 0.0001710410126672348, - "sigma_level": 114.46856639915022, - "cpk": 37.89030153444305, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.2232283353805542, - "mean_abs_diff": 0.028549259528517723, - "cosine_similarity": 0.9999403953552246, - "kl_divergence": 0.001427050064290953, - "sigma_level": 109.33200148377001, - "cpk": 36.18388818749564, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.17638635635375977, - "mean_abs_diff": 0.02896297350525856, - "cosine_similarity": 0.9999390840530396, - "kl_divergence": 0.0001846553772036104, - "sigma_level": 108.80429517535535, - "cpk": 36.00549039858327, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.16861987113952637, - "mean_abs_diff": 0.024224400520324707, - "cosine_similarity": 0.9999402761459351, - "kl_divergence": 2.34391118189284e-6, - "sigma_level": 128.46318740442206, - "cpk": 42.561733826157166, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.20461726188659668, - "mean_abs_diff": 0.03377861529588699, - "cosine_similarity": 0.9999331831932068, - "kl_divergence": 0.00023387114648966236, - "sigma_level": 94.8402052044108, - "cpk": 31.346437500955034, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.1815171241760254, - "mean_abs_diff": 0.02938883751630783, - "cosine_similarity": 0.9999383687973022, - "kl_divergence": 0.00012852325512796255, - "sigma_level": 107.84635531283064, - "cpk": 35.68466185319231, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.19804608821868896, - "mean_abs_diff": 0.039254080504179, - "cosine_similarity": 0.999911904335022, - "kl_divergence": 0.0006121267588050523, - "sigma_level": 83.23034839965909, - "cpk": 27.471188566847108, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2028665542602539, - "mean_abs_diff": 0.03341694921255112, - "cosine_similarity": 0.9999181032180786, - "kl_divergence": 0.0005078500188547661, - "sigma_level": 96.64307108681912, - "cpk": 31.94523064575194, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.16966509819030762, - "mean_abs_diff": 0.024780509993433952, - "cosine_similarity": 0.999947726726532, - "kl_divergence": 0.0005655783300468226, - "sigma_level": 126.1360790464263, - "cpk": 41.78488331819689, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.19145441055297852, - "mean_abs_diff": 0.02944508008658886, - "cosine_similarity": 0.9999592304229736, - "kl_divergence": 0.0004856583110845136, - "sigma_level": 107.8875111781858, - "cpk": 35.6977740254799, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.20439130067825317, - "mean_abs_diff": 0.028915509581565857, - "cosine_similarity": 0.9999307990074158, - "kl_divergence": 0.00023907793954586396, - "sigma_level": 110.44756254388363, - "cpk": 36.54971688521135, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.24377059936523438, - "mean_abs_diff": 0.03568901866674423, - "cosine_similarity": 0.9998804926872253, - "kl_divergence": 0.0008035140327457511, - "sigma_level": 90.22054935175149, - "cpk": 29.80519287808895, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.22080087661743164, + "mean_abs_diff": 0.029157055541872978, + "cosine_similarity": 0.9999111890792847, + "kl_divergence": 0.00032579414447661754, + "sigma_level": 108.38003583659224, + "cpk": 35.863341718487604, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.1777787208557129, + "mean_abs_diff": 0.03252752497792244, + "cosine_similarity": 0.9999105930328369, + "kl_divergence": 3.7201543263529397e-06, + "sigma_level": 99.81459433877987, + "cpk": 33.000971303716966, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.17074143886566162, + "mean_abs_diff": 0.024813607335090637, + "cosine_similarity": 0.9999530911445618, + "kl_divergence": 6.065099658800252e-06, + "sigma_level": 126.29212369865193, + "cpk": 41.83622763563623, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.17793822288513184, + "mean_abs_diff": 0.029602359980344772, + "cosine_similarity": 0.9999439120292664, + "kl_divergence": 2.1107606354716864e-05, + "sigma_level": 106.68227318083109, + "cpk": 35.29758713909199, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.21715784072875977, + "mean_abs_diff": 0.033003006130456924, + "cosine_similarity": 0.9999298453330994, + "kl_divergence": 4.5147641864858906e-07, + "sigma_level": 97.43239515014821, + "cpk": 32.20950155551228, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.16768229007720947, + "mean_abs_diff": 0.026923540979623795, + "cosine_similarity": 0.9999296069145203, + "kl_divergence": 2.4167411428152043e-05, + "sigma_level": 115.93981600656502, + "cpk": 38.38647946990311, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.1763458251953125, + "mean_abs_diff": 0.027678435668349266, + "cosine_similarity": 0.9999611973762512, + "kl_divergence": 1.2421132461965687e-06, + "sigma_level": 114.27627514935318, + "cpk": 37.8285093389394, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.2084965705871582, + "mean_abs_diff": 0.03252236545085907, + "cosine_similarity": 0.9999578595161438, + "kl_divergence": 1.444100421606142e-06, + "sigma_level": 98.27470173458114, + "cpk": 32.4918900978282, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2644195556640625, + "mean_abs_diff": 0.02975749969482422, + "cosine_similarity": 0.9999121427536011, + "kl_divergence": 0.0002420612186183089, + "sigma_level": 105.07405871764146, + "cpk": 34.764124466695144, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.21603012084960938, + "mean_abs_diff": 0.03684907406568527, + "cosine_similarity": 0.9999308586120605, + "kl_divergence": 0.0006474370152407752, + "sigma_level": 86.82147331522204, + "cpk": 28.67388353001697, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.2456355094909668, + "mean_abs_diff": 0.04065067693591118, + "cosine_similarity": 0.9999247193336487, + "kl_divergence": 0.001367731303832384, + "sigma_level": 78.56056947133877, + "cpk": 25.920728129656204, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.18509769439697266, + "mean_abs_diff": 0.03018302470445633, + "cosine_similarity": 0.9999426007270813, + "kl_divergence": 0.0005161167428984038, + "sigma_level": 104.15563778996108, + "cpk": 34.45656824761013, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.17779541015625, + "mean_abs_diff": 0.031838010996580124, + "cosine_similarity": 0.999921977519989, + "kl_divergence": 0.00017258435469219227, + "sigma_level": 99.16622380548698, + "cpk": 32.792303324828296, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.1694812774658203, + "mean_abs_diff": 0.026445960626006126, + "cosine_similarity": 0.9999368786811829, + "kl_divergence": 0.0005906903382915679, + "sigma_level": 118.52313191771943, + "cpk": 39.24650579924256, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.2208271026611328, + "mean_abs_diff": 0.0328725203871727, + "cosine_similarity": 0.9999487400054932, + "kl_divergence": 0.0007866923538006692, + "sigma_level": 96.84132305510252, + "cpk": 32.01515615449671, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18550491333007812, + "mean_abs_diff": 0.026621753349900246, + "cosine_similarity": 0.999907374382019, + "kl_divergence": 0.0002935938926612731, + "sigma_level": 117.17077206714823, + "cpk": 38.796983072900325, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.22534465789794922, + "mean_abs_diff": 0.035026710480451584, + "cosine_similarity": 0.9999264478683472, + "kl_divergence": 0.00047285014688066114, + "sigma_level": 90.85841261803856, + "cpk": 30.020931596555734, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.14307236671447754, + "mean_abs_diff": 0.02470676600933075, + "cosine_similarity": 0.999955952167511, + "kl_divergence": 3.484863188887505e-05, + "sigma_level": 128.68804300700253, + "cpk": 42.631058888436456, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.19958877563476562, + "mean_abs_diff": 0.028368674218654633, + "cosine_similarity": 0.9999251365661621, + "kl_divergence": 7.519966075083051e-05, + "sigma_level": 112.12267875571543, + "cpk": 37.1091619397265, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 5942, + "gpu_argmax": 5942, + "max_abs_diff": 0.1851489543914795, + "mean_abs_diff": 0.03547385707497597, + "cosine_similarity": 0.9999073147773743, + "kl_divergence": 0.0005868901702431452, + "sigma_level": 92.18467398700828, + "cpk": 30.455712499876228, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.2207651138305664, + "mean_abs_diff": 0.03258340433239937, + "cosine_similarity": 0.9999098777770996, + "kl_divergence": 0.0009104736533284513, + "sigma_level": 96.75204615803823, + "cpk": 31.98797279934988, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.2389364242553711, + "mean_abs_diff": 0.038926344364881516, + "cosine_similarity": 0.9999305009841919, + "kl_divergence": 0.000779499423692475, + "sigma_level": 80.85147160964138, + "cpk": 26.688219517690097, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2731236219406128, + "mean_abs_diff": 0.04174664989113808, + "cosine_similarity": 0.9998652935028076, + "kl_divergence": 0.0016339679460959167, + "sigma_level": 74.70249389609015, + "cpk": 24.64094972714047, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.24706506729125977, + "mean_abs_diff": 0.04125778004527092, + "cosine_similarity": 0.9998652935028076, + "kl_divergence": 0.00012628074726192903, + "sigma_level": 77.1027654306697, + "cpk": 25.43583106547151, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.23851919174194336, + "mean_abs_diff": 0.043194543570280075, + "cosine_similarity": 0.9997819066047668, + "kl_divergence": 0.0007928120095410873, + "sigma_level": 75.55819186163643, + "cpk": 24.914088819673893, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1764129400253296, + "mean_abs_diff": 0.026423683390021324, + "cosine_similarity": 0.9998510479927063, + "kl_divergence": 3.542603986894582e-05, + "sigma_level": 118.8965940005014, + "cpk": 39.37039083716537, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19931602478027344, + "mean_abs_diff": 0.03260093927383423, + "cosine_similarity": 0.9998522400856018, + "kl_divergence": 0.00016412521494609694, + "sigma_level": 98.43831627071472, + "cpk": 32.54534029265824, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.16838788986206055, + "mean_abs_diff": 0.025229420512914658, + "cosine_similarity": 0.9998432993888855, + "kl_divergence": 0.00022452636227061513, + "sigma_level": 124.04995771005096, + "cpk": 41.08917685771064, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.18775105476379395, + "mean_abs_diff": 0.028898026794195175, + "cosine_similarity": 0.9998990893363953, + "kl_divergence": 0.000461885527735682, + "sigma_level": 108.11494274168895, + "cpk": 35.77795520454613, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.1990138292312622, + "mean_abs_diff": 0.02971040830016136, + "cosine_similarity": 0.9999052882194519, + "kl_divergence": 0.00019171666737919304, + "sigma_level": 107.355287869335, + "cpk": 35.51929850346339, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1763872504234314, + "mean_abs_diff": 0.02600252814590931, + "cosine_similarity": 0.9998703598976135, + "kl_divergence": 5.419791774745481e-06, + "sigma_level": 121.23067932375403, + "cpk": 40.14753442864604, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19364482164382935, + "mean_abs_diff": 0.027832137420773506, + "cosine_similarity": 0.9999036192893982, + "kl_divergence": 0.00015532038116852645, + "sigma_level": 112.43699543023251, + "cpk": 37.21821831774474, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.1895672082901001, + "mean_abs_diff": 0.028663579374551773, + "cosine_similarity": 0.9999141097068787, + "kl_divergence": 0.00048805987877351045, + "sigma_level": 110.13832391530234, + "cpk": 36.44969475595692, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.1705029010772705, + "mean_abs_diff": 0.02920815348625183, + "cosine_similarity": 0.9999076724052429, + "kl_divergence": 0.0006465286747198685, + "sigma_level": 107.00006736446687, + "cpk": 35.40624958893724, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.17264866828918457, + "mean_abs_diff": 0.027726825326681137, + "cosine_similarity": 0.9998999238014221, + "kl_divergence": 9.778256838183266e-06, + "sigma_level": 113.50747960523304, + "cpk": 37.5735596967205, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.20781350135803223, + "mean_abs_diff": 0.0317465104162693, + "cosine_similarity": 0.9999275803565979, + "kl_divergence": 0.0007944463203620559, + "sigma_level": 98.32937454807063, + "cpk": 32.516323639913914, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.16794657707214355, + "mean_abs_diff": 0.025144487619400024, + "cosine_similarity": 0.9998830556869507, + "kl_divergence": 0.0014841781022014834, + "sigma_level": 124.04210451408106, + "cpk": 41.08745357459047, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.14394879341125488, + "mean_abs_diff": 0.024181175976991653, + "cosine_similarity": 0.99992835521698, + "kl_divergence": 4.590317076718935e-05, + "sigma_level": 131.6329585448339, + "cpk": 43.61239953703258, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.20485520362854004, + "mean_abs_diff": 0.032248660922050476, + "cosine_similarity": 0.9999155402183533, + "kl_divergence": 0.0010132197408340808, + "sigma_level": 97.76611926328172, + "cpk": 32.32597088527838, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.17113685607910156, + "mean_abs_diff": 0.027429627254605293, + "cosine_similarity": 0.9999209046363831, + "kl_divergence": 0.00020635141816486075, + "sigma_level": 114.89910887906412, + "cpk": 38.03706631565145, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.20895957946777344, + "mean_abs_diff": 0.03152859956026077, + "cosine_similarity": 0.9999305605888367, + "kl_divergence": 0.0012345569716387572, + "sigma_level": 100.7526951257938, + "cpk": 33.31951576032809, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2068023681640625, + "mean_abs_diff": 0.03304615244269371, + "cosine_similarity": 0.9999384880065918, + "kl_divergence": 0.0008537524363857488, + "sigma_level": 95.37599649244608, + "cpk": 31.52934802086009, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.19442176818847656, + "mean_abs_diff": 0.03274036571383476, + "cosine_similarity": 0.999926745891571, + "kl_divergence": 0.00022415574672985837, + "sigma_level": 96.70010563019095, + "cpk": 31.969535474822074, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.1620619297027588, + "mean_abs_diff": 0.02630770020186901, + "cosine_similarity": 0.9999019503593445, + "kl_divergence": 0.00015874297246952666, + "sigma_level": 119.79814893017625, + "cpk": 39.670081827825925, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2568702697753906, + "mean_abs_diff": 0.041561197489500046, + "cosine_similarity": 0.9999032020568848, + "kl_divergence": 0.0006645838048104513, + "sigma_level": 75.64519761529297, + "cpk": 24.953073788662536, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.23499298095703125, + "mean_abs_diff": 0.03136107325553894, + "cosine_similarity": 0.9999419450759888, + "kl_divergence": 0.0010564909150774027, + "sigma_level": 100.73015249089858, + "cpk": 33.31346702269047, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.19125032424926758, + "mean_abs_diff": 0.03027286008000374, + "cosine_similarity": 0.9999489188194275, + "kl_divergence": 0.0007324182535427381, + "sigma_level": 103.65502957340333, + "cpk": 34.290182007229085, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.17596006393432617, + "mean_abs_diff": 0.028325680643320084, + "cosine_similarity": 0.9999551177024841, + "kl_divergence": 0.0004250593527600838, + "sigma_level": 111.41886548737655, + "cpk": 36.87662056233914, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1672215461730957, + "mean_abs_diff": 0.02670413628220558, + "cosine_similarity": 0.9999512434005737, + "kl_divergence": 0.0003977476078045197, + "sigma_level": 118.5229880052167, + "cpk": 39.24390816638344, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.18988656997680664, + "mean_abs_diff": 0.029908394441008568, + "cosine_similarity": 0.999937117099762, + "kl_divergence": 0.00020613579089726636, + "sigma_level": 105.60082770042673, + "cpk": 34.9370799661288, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.18568384647369385, + "mean_abs_diff": 0.02834681235253811, + "cosine_similarity": 0.9999350309371948, + "kl_divergence": 0.0006665192974476394, + "sigma_level": 110.43670259979491, + "cpk": 36.55135682614585, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.20452356338500977, + "mean_abs_diff": 0.0283601526170969, + "cosine_similarity": 0.9999455809593201, + "kl_divergence": 7.353835996759626e-05, + "sigma_level": 110.17008996001424, + "cpk": 36.46299327291264, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.17694008350372314, + "mean_abs_diff": 0.029460342600941658, + "cosine_similarity": 0.999952495098114, + "kl_divergence": 0.000354042526327484, + "sigma_level": 108.24661766438967, + "cpk": 35.81645735131439, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.17319059371948242, + "mean_abs_diff": 0.025698617100715637, + "cosine_similarity": 0.9999451041221619, + "kl_divergence": 0.0012082822780696116, + "sigma_level": 121.84608568951509, + "cpk": 40.35442223805871, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.18508267402648926, + "mean_abs_diff": 0.026767021045088768, + "cosine_similarity": 0.9999246597290039, + "kl_divergence": 8.510791162766133e-06, + "sigma_level": 117.67908059360465, + "cpk": 38.96386699563357, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.23066043853759766, + "mean_abs_diff": 0.036110661923885345, + "cosine_similarity": 0.9999273419380188, + "kl_divergence": 0.0004994907989914782, + "sigma_level": 88.93142368056236, + "cpk": 29.3761935122759, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.18856239318847656, + "mean_abs_diff": 0.030129026621580124, + "cosine_similarity": 0.9999247789382935, + "kl_divergence": 0.0008435059720457446, + "sigma_level": 104.45399344155254, + "cpk": 34.55573971808994, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.16795825958251953, + "mean_abs_diff": 0.0249911118298769, + "cosine_similarity": 0.9999393820762634, + "kl_divergence": 0.0001998253593928442, + "sigma_level": 124.82156616889559, + "cpk": 41.34723624638959, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.17892026901245117, + "mean_abs_diff": 0.028010983020067215, + "cosine_similarity": 0.9999496936798096, + "kl_divergence": 4.5661537655344266e-05, + "sigma_level": 112.21705530380771, + "cpk": 37.143742598712834, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.23161697387695312, + "mean_abs_diff": 0.03567603975534439, + "cosine_similarity": 0.9999344348907471, + "kl_divergence": 0.0002818625074985845, + "sigma_level": 90.15571250675596, + "cpk": 29.78387093695512, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.17027735710144043, + "mean_abs_diff": 0.026973217725753784, + "cosine_similarity": 0.9999549388885498, + "kl_divergence": 0.0005629856415008736, + "sigma_level": 116.05363658933409, + "cpk": 38.423683862478896, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.1832141876220703, + "mean_abs_diff": 0.02868635207414627, + "cosine_similarity": 0.999939501285553, + "kl_divergence": 0.0006119540337820587, + "sigma_level": 109.6258088123588, + "cpk": 36.27987255845257, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.2024369239807129, + "mean_abs_diff": 0.0438840389251709, + "cosine_similarity": 0.9999357461929321, + "kl_divergence": 0.00047113813301219134, + "sigma_level": 75.99682576801438, + "cpk": 25.054354617655378, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.18555545806884766, + "mean_abs_diff": 0.025885246694087982, + "cosine_similarity": 0.9999472498893738, + "kl_divergence": 0.000672849308967897, + "sigma_level": 121.69151806777218, + "cpk": 40.301338108777195, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.16499638557434082, + "mean_abs_diff": 0.02714230865240097, + "cosine_similarity": 0.9999595284461975, + "kl_divergence": 0.000296325939274267, + "sigma_level": 114.99287429648993, + "cpk": 38.070860424914805, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 9079, + "gpu_argmax": 3589, + "max_abs_diff": 0.17669677734375, + "mean_abs_diff": 0.02647567354142666, + "cosine_similarity": 0.9999456405639648, + "kl_divergence": 0.00043440203458147177, + "sigma_level": 119.37552964782644, + "cpk": 39.52846425329292, + "verdict": "WarnArgmax" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.15739870071411133, + "mean_abs_diff": 0.0278735663741827, + "cosine_similarity": 0.999921977519989, + "kl_divergence": 0.0001710410126672348, + "sigma_level": 114.46856639915022, + "cpk": 37.89030153444305, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.2232283353805542, + "mean_abs_diff": 0.028549259528517723, + "cosine_similarity": 0.9999403953552246, + "kl_divergence": 0.001427050064290953, + "sigma_level": 109.33200148377001, + "cpk": 36.18388818749564, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.17638635635375977, + "mean_abs_diff": 0.02896297350525856, + "cosine_similarity": 0.9999390840530396, + "kl_divergence": 0.0001846553772036104, + "sigma_level": 108.80429517535535, + "cpk": 36.00549039858327, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.16861987113952637, + "mean_abs_diff": 0.024224400520324707, + "cosine_similarity": 0.9999402761459351, + "kl_divergence": 2.34391118189284e-06, + "sigma_level": 128.46318740442206, + "cpk": 42.561733826157166, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.20461726188659668, + "mean_abs_diff": 0.03377861529588699, + "cosine_similarity": 0.9999331831932068, + "kl_divergence": 0.00023387114648966236, + "sigma_level": 94.8402052044108, + "cpk": 31.346437500955034, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.1815171241760254, + "mean_abs_diff": 0.02938883751630783, + "cosine_similarity": 0.9999383687973022, + "kl_divergence": 0.00012852325512796255, + "sigma_level": 107.84635531283064, + "cpk": 35.68466185319231, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.19804608821868896, + "mean_abs_diff": 0.039254080504179, + "cosine_similarity": 0.999911904335022, + "kl_divergence": 0.0006121267588050523, + "sigma_level": 83.23034839965909, + "cpk": 27.471188566847108, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2028665542602539, + "mean_abs_diff": 0.03341694921255112, + "cosine_similarity": 0.9999181032180786, + "kl_divergence": 0.0005078500188547661, + "sigma_level": 96.64307108681912, + "cpk": 31.94523064575194, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.16966509819030762, + "mean_abs_diff": 0.024780509993433952, + "cosine_similarity": 0.999947726726532, + "kl_divergence": 0.0005655783300468226, + "sigma_level": 126.1360790464263, + "cpk": 41.78488331819689, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.19145441055297852, + "mean_abs_diff": 0.02944508008658886, + "cosine_similarity": 0.9999592304229736, + "kl_divergence": 0.0004856583110845136, + "sigma_level": 107.8875111781858, + "cpk": 35.6977740254799, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.20439130067825317, + "mean_abs_diff": 0.028915509581565857, + "cosine_similarity": 0.9999307990074158, + "kl_divergence": 0.00023907793954586396, + "sigma_level": 110.44756254388363, + "cpk": 36.54971688521135, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.24377059936523438, + "mean_abs_diff": 0.03568901866674423, + "cosine_similarity": 0.9998804926872253, + "kl_divergence": 0.0008035140327457511, + "sigma_level": 90.22054935175149, + "cpk": 29.80519287808895, + "verdict": "Pass" + } ] + } } diff --git a/scripts/check_model_parity.sh b/scripts/check_model_parity.sh index 4a75d10585..1e3927f33c 100755 --- a/scripts/check_model_parity.sh +++ b/scripts/check_model_parity.sh @@ -39,7 +39,13 @@ if "min_cosine" not in rule or not str(rule.get("basis") or "").strip(): mc = float(rule["min_cosine"]); basis = str(rule["basis"]) try: d = json.load(open(f, encoding="utf-8")) except Exception as e: print(f"FAIL {model}: unreadable apr parity output ({e})"); sys.exit(1) -rows = d.get("metrics") if isinstance(d, dict) else None +# A v2 receipt (apr-parity-receipt/v2, #3577) EMBEDS the raw `apr parity --json` document under +# `raw`, so the readings live at `raw.metrics`. A fresh `apr parity` run is the raw document itself +# and carries them at the top level. Those are two different INPUTS — a tool's output and an archived +# receipt that quotes it — not two spellings of one, so the rule is stated once: look inside the +# envelope when there is one. Anything else is still refused by the line below. +raw = d.get("raw") if isinstance(d, dict) and isinstance(d.get("raw"), dict) else d +rows = raw.get("metrics") if isinstance(raw, dict) else None if not isinstance(rows, list) or not rows: print(f"FAIL {model}: no per-position metrics in the output"); sys.exit(1) cos = [(r.get("position"), float(r.get("cosine_similarity"))) for r in rows if r.get("cosine_similarity") is not None] if len(cos) < minpos: print(f"FAIL {model}: {len(cos)} positions < min_positions {minpos} (I8: an autoregressive gate validates over >= 64 positions)"); sys.exit(1) @@ -172,9 +178,16 @@ if [ "${1:-}" = "--self-test" ]; then row 0 "the gx10 7B record is GREEN (min 0.9985)" judge "$G/qwen2.5-coder-7b-instruct-q4_k_m.json" qwen2.5-coder-7b-instruct python3 - "$L/qwen2.5-coder-7b-instruct-q4_k_m.json" "$TD/twin.json" "$TD/short.json" <<'PY' import json, sys -d = json.load(open(sys.argv[1])); m = d["metrics"] -m[40]["cosine_similarity"] = 0.5; json.dump(d, open(sys.argv[2], "w")) # must-RED twin: one position at 0.5 -d2 = json.load(open(sys.argv[1])); d2["metrics"] = d2["metrics"][:20]; json.dump(d2, open(sys.argv[3], "w")) # fewer than 64 positions +# The committed records are apr-parity-receipt/v2 (#3577): the raw `apr parity --json` document is +# embedded under `raw`. Mutate the readings where they actually live, or the twin is built from a +# KeyError and the must-RED control never fires. +def rows(doc): return doc["raw"]["metrics"] if isinstance(doc.get("raw"), dict) else doc["metrics"] +def put(doc, v): + (doc["raw"] if isinstance(doc.get("raw"), dict) else doc)["metrics"] = v + return doc +d = json.load(open(sys.argv[1])); rows(d)[40]["cosine_similarity"] = 0.5 +json.dump(d, open(sys.argv[2], "w")) # must-RED twin: one position at 0.5 +d2 = json.load(open(sys.argv[1])); json.dump(put(d2, rows(d2)[:20]), open(sys.argv[3], "w")) # fewer than 64 positions PY [ -f "$ROOT/tests/fixtures/parity/defective/one-position-at-0.5.json" ] || cp "$TD/twin.json" "$ROOT/tests/fixtures/parity/defective/one-position-at-0.5.json" row 1 "the must-RED twin (7B with position 40 forced to 0.5) is RED naming the position" judge "$TD/twin.json" qwen2.5-coder-7b-instruct diff --git a/scripts/parity_receipt_denominator.sh b/scripts/parity_receipt_denominator.sh new file mode 100755 index 0000000000..8e5b1780dc --- /dev/null +++ b/scripts/parity_receipt_denominator.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# parity_receipt_denominator.sh — PMAT-3577 / #3577. +# +# THE PREDICATE, not the extractor. `evidence/parity/EXPECTED_RECEIPTS` records how many logit-parity +# receipts the tree holds; this script recomputes that number from the tree itself, by a rule written +# INDEPENDENTLY of the Rust extractor. Two implementations of one question is the whole point: an +# extractor checked against a number the extractor produced proves nothing. +# +# The rule: a file under `evidence/parity/**` is a receipt iff it is JSON whose top-level `schema` is +# `apr-parity-receipt/v2`. Anything else is not counted, and an UNMIGRATED legacy record — no schema but a +# top-level `metrics[]` or `parity` — is REFUSED, because a record the extractor cannot see is a record no +# shape can refuse. +# +# The universe is `git ls-files`, never `find`: `.claude/worktrees/lane-*` holds full clones at other +# commits, and `find` returns them (aprender#3579). +# +# bash scripts/parity_receipt_denominator.sh # verify against EXPECTED_RECEIPTS +# bash scripts/parity_receipt_denominator.sh --print # print the measured count and exit 0 +# bash scripts/parity_receipt_denominator.sh --self-test +# +# Exit: 0 agree · 1 disagree (or an unmigrated record) · 2 usage / the file is missing. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +EXPECTED_FILE="evidence/parity/EXPECTED_RECEIPTS" +SCHEMA="apr-parity-receipt/v2" + +count_and_check() { + local root=$1 records=0 unmigrated=() + local f + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$(classify "$root/$f")" in + record) records=$((records + 1)) ;; + legacy) unmigrated+=("$f") ;; + *) : ;; + esac + done < <(cd "$root" && git ls-files 'evidence/parity/*.json' 'evidence/parity/**/*.json' 2>/dev/null | sort -u) + if [ "${#unmigrated[@]}" -gt 0 ]; then + printf 'FAIL %s unmigrated legacy record(s) - no schema, but a top-level metrics[]/parity:\n' \ + "${#unmigrated[@]}" >&2 + printf ' %s\n' "${unmigrated[@]}" >&2 + return 1 + fi + printf '%s\n' "$records" +} + +# record | legacy | other — one file, by its own content. +classify() { + python3 - "$1" "$SCHEMA" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception: + print("other"); raise SystemExit(0) +if not isinstance(d, dict): + print("other"); raise SystemExit(0) +if d.get("schema") == sys.argv[2]: + print("record") +elif isinstance(d.get("metrics"), list) or "parity" in d: + print("legacy") +else: + print("other") +PY +} + +expected_of() { + grep -vE '^\s*(#|$)' "$1/$EXPECTED_FILE" | head -1 | tr -d '[:space:]' +} + +# Remove a directory this script created, and NOTHING else. The validation is here, immediately above +# the `rm`, rather than at the call site: a guard the reader has to go and find is a guard that gets +# moved away from what it protects (bashrs SEC011). +discard_tempdir() { + dir=$1 + [ -n "$dir" ] || return 0 + [ -d "$dir" ] || return 0 + case "$dir" in + /tmp/*|/var/folders/*) : ;; + *) printf 'refusing to remove %s: not a temp directory this script made\n' "$dir" >&2; return 0 ;; + esac + rm -rf -- "$dir" +} + +self_test() { + local rc=0 td + td=$(mktemp -d) + [ -n "$td" ] && [ -d "$td" ] && [ "${#td}" -gt 8 ] || { printf 'FAIL mktemp -d gave %s\n' "${td:-}" >&2; return 2; } + trap 'discard_tempdir "$td"' RETURN + # hooksPath off: the self-test's throwaway repos must not run the developer's global pre-commit hook. + (cd "$td" && git init -q . && git config user.email t@t && git config user.name t \ + && git config core.hooksPath /dev/null) + mkdir -p "$td/evidence/parity/l0-1/lambda" + printf '{"schema":"%s","host":"h"}\n' "$SCHEMA" > "$td/evidence/parity/l0-1/lambda/a.json" + printf '{"seed":1}\n' > "$td/evidence/parity/props-x.json" + printf '1\n' > "$td/$EXPECTED_FILE" + (cd "$td" && git add -A && git commit -qm t) + + # ONE record, one unrelated document, denominator 1 -> agree. + if out=$(count_and_check "$td" 2>&1) && [ "$out" = 1 ]; then + printf 'ok a record is counted and an unrelated document is not (measured %s)\n' "$out" + else + printf 'FAIL expected 1, got %s\n' "$out"; rc=1 + fi + + # A legacy record appears -> REFUSED, never counted, never skipped. + printf '{"model":"./m.gguf","parity":true,"metrics":[]}\n' > "$td/evidence/parity/l0-1/lambda/legacy.json" + (cd "$td" && git add -A && git commit -qm legacy) + if count_and_check "$td" >/dev/null 2>&1; then + printf 'FAIL an unmigrated legacy record did not refuse\n'; rc=1 + else + printf 'ok an unmigrated legacy record is refused by name\n' + fi + rm "$td/evidence/parity/l0-1/lambda/legacy.json" + (cd "$td" && git add -A && git commit -qm rm) + + # A receipt added without bumping the denominator -> disagree. THE falsifier the row names. + printf '{"schema":"%s","host":"h2"}\n' "$SCHEMA" > "$td/evidence/parity/l0-1/lambda/b.json" + (cd "$td" && git add -A && git commit -qm add) + if verify "$td" >/dev/null 2>&1; then + printf 'FAIL a receipt added without bumping the denominator passed\n'; rc=1 + else + printf 'ok a receipt added without bumping the denominator disagrees\n' + fi + + # And bumping it makes them agree again — BOTH directions, or the control proves nothing. + printf '2\n' > "$td/$EXPECTED_FILE" + (cd "$td" && git add -A && git commit -qm bump) + if verify "$td" >/dev/null 2>&1; then + printf 'ok bumping the denominator makes them agree\n' + else + printf 'FAIL the denominator was bumped and they still disagree\n'; rc=1 + fi + return "$rc" +} + +verify() { + local root=$1 measured expected + [ -f "$root/$EXPECTED_FILE" ] || { printf 'FAIL %s is missing\n' "$EXPECTED_FILE" >&2; return 2; } + measured=$(count_and_check "$root") || return 1 + expected=$(expected_of "$root") + if [ "$measured" = "$expected" ]; then + printf 'PASS %s receipt(s) under evidence/parity/**, and %s says %s.\n' \ + "$measured" "$EXPECTED_FILE" "$expected" + return 0 + fi + printf 'FAIL %s says %s; the tree holds %s.\n' "$EXPECTED_FILE" "$expected" "$measured" >&2 + printf ' Unknown{WrongCorpus}: update the denominator in the SAME commit as the receipt.\n' >&2 + return 1 +} + +case "${1:-}" in + --self-test) self_test ;; + --print) count_and_check "$ROOT" ;; + "") verify "$ROOT" ;; + *) printf 'usage: %s [--print|--self-test]\n' "$(basename "$0")" >&2; exit 2 ;; +esac diff --git a/scripts/tree_reader_tests.txt b/scripts/tree_reader_tests.txt index e93a263eb2..95af93bfdf 100644 --- a/scripts/tree_reader_tests.txt +++ b/scripts/tree_reader_tests.txt @@ -68,6 +68,7 @@ aprender-contracts-cli --test ont4_relations_gate aprender-contracts-cli --test ont4b2_code_lean_w3c aprender-contracts-cli --test ont4b_shapes_gate aprender-contracts-cli --test ont4c1_model_receipts +aprender-contracts-cli --test ont4c3_parity_receipts aprender-contracts-cli --test ont6_lint_verdict aprender-contracts-cli --test ont6b_kind_default aprender-contracts-cli --test ont_entity_properties diff --git a/tests/fixtures/ont/parity-denominator-drift/contracts/parity-receipt-v2.yaml b/tests/fixtures/ont/parity-denominator-drift/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/tests/fixtures/ont/parity-denominator-drift/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/tests/fixtures/ont/parity-denominator-drift/evidence/parity/EXPECTED_RECEIPTS b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1 @@ +1 diff --git a/tests/fixtures/ont/parity-denominator-drift/evidence/parity/a.json b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/a.json new file mode 100644 index 0000000000..94f8dae207 --- /dev/null +++ b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/a.json @@ -0,0 +1,45 @@ +{ + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "no oracle arm exists for this cell: apr-CPU vs apr-CUDA on one binary" + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured." + ], + "result": { + "positions": 78, + "parity": true, + "min_cosine": 0.9986, + "threshold": 0.98, + "verdict": "PASS" + }, + "raw": { + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + }, + { + "position": 1, + "cosine_similarity": 0.9991 + } + ] + } +} diff --git a/tests/fixtures/ont/parity-denominator-drift/evidence/parity/b.json b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/b.json new file mode 100644 index 0000000000..94f8dae207 --- /dev/null +++ b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/b.json @@ -0,0 +1,45 @@ +{ + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "no oracle arm exists for this cell: apr-CPU vs apr-CUDA on one binary" + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured." + ], + "result": { + "positions": 78, + "parity": true, + "min_cosine": 0.9986, + "threshold": 0.98, + "verdict": "PASS" + }, + "raw": { + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + }, + { + "position": 1, + "cosine_similarity": 0.9991 + } + ] + } +} diff --git a/tests/fixtures/ont/parity-denominator-drift/evidence/parity/thresholds.yaml b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/thresholds.yaml new file mode 100644 index 0000000000..4eccbd69e8 --- /dev/null +++ b/tests/fixtures/ont/parity-denominator-drift/evidence/parity/thresholds.yaml @@ -0,0 +1,5 @@ +schema: apr-parity-thresholds/v1 +min_positions: 64 +default: + min_cosine: 0.98 + basis: 'fixture' diff --git a/tests/fixtures/ont/parity-green/contracts/parity-receipt-v2.yaml b/tests/fixtures/ont/parity-green/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/tests/fixtures/ont/parity-green/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/tests/fixtures/ont/parity-green/evidence/parity/EXPECTED_RECEIPTS b/tests/fixtures/ont/parity-green/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/fixtures/ont/parity-green/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1 @@ +1 diff --git a/tests/fixtures/ont/parity-green/evidence/parity/receipt.json b/tests/fixtures/ont/parity-green/evidence/parity/receipt.json new file mode 100644 index 0000000000..94f8dae207 --- /dev/null +++ b/tests/fixtures/ont/parity-green/evidence/parity/receipt.json @@ -0,0 +1,45 @@ +{ + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "no oracle arm exists for this cell: apr-CPU vs apr-CUDA on one binary" + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured." + ], + "result": { + "positions": 78, + "parity": true, + "min_cosine": 0.9986, + "threshold": 0.98, + "verdict": "PASS" + }, + "raw": { + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + }, + { + "position": 1, + "cosine_similarity": 0.9991 + } + ] + } +} diff --git a/tests/fixtures/ont/parity-green/evidence/parity/thresholds.yaml b/tests/fixtures/ont/parity-green/evidence/parity/thresholds.yaml new file mode 100644 index 0000000000..4eccbd69e8 --- /dev/null +++ b/tests/fixtures/ont/parity-green/evidence/parity/thresholds.yaml @@ -0,0 +1,5 @@ +schema: apr-parity-thresholds/v1 +min_positions: 64 +default: + min_cosine: 0.98 + basis: 'fixture' diff --git a/tests/fixtures/ont/parity-nocomparator/contracts/parity-receipt-v2.yaml b/tests/fixtures/ont/parity-nocomparator/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/tests/fixtures/ont/parity-nocomparator/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/tests/fixtures/ont/parity-nocomparator/evidence/parity/EXPECTED_RECEIPTS b/tests/fixtures/ont/parity-nocomparator/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/fixtures/ont/parity-nocomparator/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1 @@ +1 diff --git a/tests/fixtures/ont/parity-nocomparator/evidence/parity/receipt.json b/tests/fixtures/ont/parity-nocomparator/evidence/parity/receipt.json new file mode 100644 index 0000000000..40bc9bc99b --- /dev/null +++ b/tests/fixtures/ont/parity-nocomparator/evidence/parity/receipt.json @@ -0,0 +1,41 @@ +{ + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured." + ], + "result": { + "positions": 78, + "parity": true, + "min_cosine": 0.9986, + "threshold": 0.98, + "verdict": "PASS" + }, + "raw": { + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + }, + { + "position": 1, + "cosine_similarity": 0.9991 + } + ] + } +} diff --git a/tests/fixtures/ont/parity-nocomparator/evidence/parity/thresholds.yaml b/tests/fixtures/ont/parity-nocomparator/evidence/parity/thresholds.yaml new file mode 100644 index 0000000000..4eccbd69e8 --- /dev/null +++ b/tests/fixtures/ont/parity-nocomparator/evidence/parity/thresholds.yaml @@ -0,0 +1,5 @@ +schema: apr-parity-thresholds/v1 +min_positions: 64 +default: + min_cosine: 0.98 + basis: 'fixture' diff --git a/tests/fixtures/ont/parity-unknownkind/contracts/parity-receipt-v2.yaml b/tests/fixtures/ont/parity-unknownkind/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/tests/fixtures/ont/parity-unknownkind/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/tests/fixtures/ont/parity-unknownkind/evidence/parity/EXPECTED_RECEIPTS b/tests/fixtures/ont/parity-unknownkind/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/fixtures/ont/parity-unknownkind/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1 @@ +1 diff --git a/tests/fixtures/ont/parity-unknownkind/evidence/parity/receipt.json b/tests/fixtures/ont/parity-unknownkind/evidence/parity/receipt.json new file mode 100644 index 0000000000..81f0609c13 --- /dev/null +++ b/tests/fixtures/ont/parity-unknownkind/evidence/parity/receipt.json @@ -0,0 +1,45 @@ +{ + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "oracle", + "reason": "no oracle arm exists for this cell: apr-CPU vs apr-CUDA on one binary" + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured." + ], + "result": { + "positions": 78, + "parity": true, + "min_cosine": 0.9986, + "threshold": 0.98, + "verdict": "PASS" + }, + "raw": { + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + }, + { + "position": 1, + "cosine_similarity": 0.9991 + } + ] + } +} diff --git a/tests/fixtures/ont/parity-unknownkind/evidence/parity/thresholds.yaml b/tests/fixtures/ont/parity-unknownkind/evidence/parity/thresholds.yaml new file mode 100644 index 0000000000..4eccbd69e8 --- /dev/null +++ b/tests/fixtures/ont/parity-unknownkind/evidence/parity/thresholds.yaml @@ -0,0 +1,5 @@ +schema: apr-parity-thresholds/v1 +min_positions: 64 +default: + min_cosine: 0.98 + basis: 'fixture' diff --git a/tests/fixtures/ont/parity-unmigrated/contracts/parity-receipt-v2.yaml b/tests/fixtures/ont/parity-unmigrated/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/tests/fixtures/ont/parity-unmigrated/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/tests/fixtures/ont/parity-unmigrated/evidence/parity/EXPECTED_RECEIPTS b/tests/fixtures/ont/parity-unmigrated/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/fixtures/ont/parity-unmigrated/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1 @@ +1 diff --git a/tests/fixtures/ont/parity-unmigrated/evidence/parity/legacy.json b/tests/fixtures/ont/parity-unmigrated/evidence/parity/legacy.json new file mode 100644 index 0000000000..8b939aa45d --- /dev/null +++ b/tests/fixtures/ont/parity-unmigrated/evidence/parity/legacy.json @@ -0,0 +1,13 @@ +{ + "model": "./m.gguf", + "tokens": 78, + "passed": 78, + "failed": 0, + "parity": true, + "metrics": [ + { + "position": 0, + "cosine_similarity": 0.9986 + } + ] +} diff --git a/tests/fixtures/ont/parity-unmigrated/evidence/parity/thresholds.yaml b/tests/fixtures/ont/parity-unmigrated/evidence/parity/thresholds.yaml new file mode 100644 index 0000000000..4eccbd69e8 --- /dev/null +++ b/tests/fixtures/ont/parity-unmigrated/evidence/parity/thresholds.yaml @@ -0,0 +1,5 @@ +schema: apr-parity-thresholds/v1 +min_positions: 64 +default: + min_cosine: 0.98 + basis: 'fixture'