diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92fa61de8b..f678438abc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -445,7 +445,9 @@ jobs: case "${GITHUB_EVENT_NAME}" in pull_request) git fetch --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" - git diff --name-only "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" + # --no-renames (#3664): a rename must list BOTH paths, or a source + # moved into docs/ reads as docs-only and runs zero tests. + git diff --no-renames --name-only "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" printf 'touched (tip vs tip): %s path(s)\n' "$(wc -l < "$RUNNER_TEMP/touched.txt" | tr -d ' ')" args+=(--diff-from "$RUNNER_TEMP/touched.txt") ;; @@ -827,6 +829,17 @@ jobs: run: | printf 'workspace-test REUSED: %s\n' "${{ steps.tier.outputs.reason }}" printf 'cited PR head: %s\n' "${{ steps.tier.outputs.cite }}" + # #3658 NONE: the tier step proved the change gives workspace-test nothing + # to measure -- every touched path is under docs/roadmaps/ or docs/audits/ + # and present at HEAD (no Rust source, test or tree reader reads them), or + # the change lands no path at all. Nothing is built or run; this step + # records why, so the skip is auditable in the log. + - name: "None: nothing for workspace-test to measure (#3658)" + if: steps.tier.outputs.tier == 'none' + env: + REASON: ${{ steps.tier.outputs.reason }} + run: | + printf 'workspace-test NOT RUN: %s\n' "$REASON" - name: Build.rs crate-root escape check (v0.31.1 yank guard) if: matrix.shard == 1 # Static Poka-Yoke: flags build.rs files that panic on files outside @@ -1211,6 +1224,14 @@ jobs: run: bash scripts/derive_model_manifest.sh --check - name: C14 model-parity case table (L0-1a, #2971) run: bash scripts/check_model_parity.sh --self-test + # #3577: the parity-receipt denominator is the SECOND, independent reader of the + # receipt corpus — the extractor is the first, and they share no code so a defect + # in either is caught by the other rather than by nobody. It shipped unwired, which + # `check_guards_are_wired.sh` caught: a refusal nothing runs is not a refusal, and + # `parity-receipt-v1.yaml`'s own equations name "both readers are wired into CI" as + # a precondition. Its 4-case self-test is cargo-free and model-free, so it runs here. + - name: Parity-receipt denominator case table (#3577) + run: bash scripts/parity_receipt_denominator.sh --self-test # F-1 (#3022, #3024): a model file the user NAMED is the model that answers, or # the command refuses — `apr chat` silently loaded its toy demo model for a # `model.safetensors.index.json` (exit 0, zero tokens, the real path in the banner) diff --git a/Cargo.lock b/Cargo.lock index 5d45f73d89..eba977fa2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,6 +265,7 @@ dependencies = [ "aprender-present-core", "aprender-present-terminal", "aprender-profile", + "aprender-quant", "aprender-rag-cli", "aprender-registry", "aprender-serve", @@ -1359,6 +1360,7 @@ dependencies = [ "serde_json", "serde_yaml_ng", "serial_test", + "sha2 0.10.9", "smallvec", "sysinfo 0.32.1", "tempfile", diff --git a/README.md b/README.md index 7980187815..641c085126 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **79** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1801** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1828** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **110** CLI commands | `apr --help` | | Book CLI chapters | **112** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **71** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -262,7 +262,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1801 contracts across inference, training, quantization, attention, FFN, +The tree carries 1828 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd b/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd new file mode 100644 index 0000000000..b294234cdd --- /dev/null +++ b/ci/explicit-test-commands.d/420-aprender-contracts-cli-ont4c3-parity-receipts.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts diff --git a/ci/explicit-test-commands.d/430-aprender-contracts-cli-ont-refusal-receipt.cmd b/ci/explicit-test-commands.d/430-aprender-contracts-cli-ont-refusal-receipt.cmd new file mode 100644 index 0000000000..ab720e9323 --- /dev/null +++ b/ci/explicit-test-commands.d/430-aprender-contracts-cli-ont-refusal-receipt.cmd @@ -0,0 +1 @@ +cargo test -p aprender-contracts-cli --test ont_refusal_receipt diff --git a/contracts/apr-dogfood-coverage-v1.yaml b/contracts/apr-dogfood-coverage-v1.yaml index 1395d93564..5124a5c5be 100644 --- a/contracts/apr-dogfood-coverage-v1.yaml +++ b/contracts/apr-dogfood-coverage-v1.yaml @@ -56,6 +56,12 @@ metadata: # drifted, +45 lines at ce712eae0 and -39 at bd475b06f, so every citation was # 6 lines short of its arm), and two new rows for `apr test llm bench` and its # `--band` mode, which shipped with no ledger entry at all. + # PARTIAL RE-AUDIT, 0.69 release batch #3669 (G2.1). Same rule, same reason: + # `measured_commit` does not move, and a batch-branch SHA would not survive the + # squash merge anyway. #3604 inserted 4 lines (`--revalidate`) after line 138 of + # crates/apr-cli/src/commands_enum.rs. All 64 citations of that file in 55 rows + # were re-derived: 42 at lines >= 139 shifted by +4, 22 at <= 138 unchanged, and + # every cited line's text was checked identical between 79a3af79d and the batch. ledger: docs/audits/surface_audit.csv overall: diff --git a/contracts/apr-inspect-dtype-naming-v1.yaml b/contracts/apr-inspect-dtype-naming-v1.yaml index 035b1567a6..4376794e38 100644 --- a/contracts/apr-inspect-dtype-naming-v1.yaml +++ b/contracts/apr-inspect-dtype-naming-v1.yaml @@ -1,5 +1,5 @@ metadata: - version: 1.0.0 + version: 1.1.0 created: '2026-04-05' author: PAIML Engineering description: "apr inspect/rosetta dtype naming contract — DType column must render human-readable GGML type names (F32, Q4_K, Q6_K), never raw integer discriminants" @@ -7,6 +7,7 @@ metadata: - "paiml/aprender#619 (inspect/rosetta: DType column shows integer IDs instead of names)" - "paiml/aprender#605 (DType IDs historical pattern)" - "paiml/aprender#603 (quantization field shows '0' — downstream of same root cause)" + - "paiml/aprender#3601 (names read from trueno_quant::TRAITS; ids 31-42 — TQ1_0, TQ2_0, MXFP4, NVFP4, Q1_0, Q2_0 — are named, removed ids stay 'unknown')" registry: true tags: - cli @@ -24,7 +25,7 @@ equations: invariants: - "TensorInfo.dtype is ALWAYS a human-readable name from the GGML canonical set" - "TensorInfo.dtype NEVER parses as an integer via str::parse::" - - "GGML_NAMES = {F32, F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, Q2_K, Q3_K, Q4_K, Q5_K, Q6_K, Q8_K, IQ2_XXS, IQ2_XS, IQ3_XXS, IQ1_S, IQ4_NL, IQ3_S, IQ2_S, IQ4_XS, I8, I16, BF16, I32, I64, F64, IQ1_M, unknown}" + - "GGML_NAMES = {F32, F16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, Q2_K, Q3_K, Q4_K, Q5_K, Q6_K, Q8_K, IQ2_XXS, IQ2_XS, IQ3_XXS, IQ1_S, IQ4_NL, IQ3_S, IQ2_S, IQ4_XS, I8, I16, BF16, I32, I64, F64, IQ1_M, TQ1_0, TQ2_0, MXFP4, NVFP4, Q1_0, Q2_0, unknown}" cross_cmd_consistency: formula: "∀ tensor T in file F: dtype(inspect F, T) = dtype(tensors F, T)" @@ -81,8 +82,8 @@ falsification_tests: kani_harnesses: - id: KANI-INSPECT-DTYPE-001 - obligation: "Every GGML u32 discriminant ∈ [0,30] maps to a non-integer name" - bound: 31 + obligation: "Every GGML u32 discriminant ∈ [0,42] (GGML_TYPE_COUNT = 43) maps to a non-integer name" + bound: 43 strategy: bounded_int qa_gate: diff --git a/contracts/binding.yaml b/contracts/binding.yaml index c4a9520ec8..1391a732f4 100644 --- a/contracts/binding.yaml +++ b/contracts/binding.yaml @@ -764,7 +764,7 @@ bindings: notes: "P = V*d + L*(d_attn + d_ffn + d_norm) + d_final + V*d. Per-layer terms come in as data\ \ (&[LayerParams]) so a hybrid family can mix layer kinds. QE2E-INV-001 (P(9B) in [9.0B, 9.2B])\ \ is NOT discharged: dense/GQA accounting gives 8.21B and the Gated DeltaNet d_attn needs\ - \ inner_size/state_size/conv_kernel/group_count, which ModelConstraints does not carry (#3347)." + \ inner_size/state_size/conv_kernel/group_count, which the 9b descriptor disagrees with itself (inner_size 2048 at hidden_dim 4096, and group_count 8 fails group_count*state_size == inner_size), so P(9B) computes to 8.345B; settling it needs a real Qwen3.5-9B GGUF (#3346)." - contract: qwen35-e2e-verification-v1.yaml equation: flops_per_token module_path: aprender::format::model_arithmetic diff --git a/contracts/census.json b/contracts/census.json index 016e5b245e..8162052f50 100644 --- a/contracts/census.json +++ b/contracts/census.json @@ -1,18 +1,18 @@ { "schema": "ont.paiml.dev/census/v1alpha1", "git_sha": null, - "n_files": 1801, - "n_parsed": 1801, + "n_files": 1828, + "n_parsed": 1828, "n_parse_errors": 0, "parse_errors": [], "quarantined_n": 0, "by_kind": { "beat-benchmark": 24, "corpus-assembly": 1, - "kernel": 363, + "kernel": 387, "model-family": 28, "model-family-variant": 1, - "pattern": 87, + "pattern": 90, "pretraining-corpus": 2, "registry": 519, "schema": 766, @@ -22,14 +22,15 @@ }, "by_entity_type": { "gguf": 2, + "json": 3, "pv-contract": 1 }, "by_anchoring": { - "unanchored": 1798, - "class": 2, - "instance": 1 + "unanchored": 1822, + "class": 4, + "instance": 2 }, - "id_set_sha256": "e624814d2c82e8a53ab24faac0b5281c0dad15618d9f14e970c740c8391cb712", + "id_set_sha256": "69d8018f1f9cc7c7f16fb61566d2a405c1c6ab652c90936cc55398a4549825fe", "declared_external": [ { "name": "provable-contracts", diff --git a/contracts/contracts.nt b/contracts/contracts.nt index dc9a62ac7d..7646da81bd 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -7393,6 +7393,102 @@ . "contracts/crux-N-17-v1.yaml"^^ . "crux-N-17-v1"^^ . + . + . + "contracts/crux-O-01-v1.yaml"^^ . + "crux-O-01-v1"^^ . + . + . + "contracts/crux-O-02-v1.yaml"^^ . + "crux-O-02-v1"^^ . + . + . + "contracts/crux-O-03-v1.yaml"^^ . + "crux-O-03-v1"^^ . + . + . + "contracts/crux-O-04-v1.yaml"^^ . + "crux-O-04-v1"^^ . + . + . + "contracts/crux-O-05-v1.yaml"^^ . + "crux-O-05-v1"^^ . + . + . + "contracts/crux-O-06-v1.yaml"^^ . + "crux-O-06-v1"^^ . + . + . + "contracts/crux-O-07-v1.yaml"^^ . + "crux-O-07-v1"^^ . + . + . + "contracts/crux-O-08-v1.yaml"^^ . + "crux-O-08-v1"^^ . + . + . + "contracts/crux-O-09-v1.yaml"^^ . + "crux-O-09-v1"^^ . + . + . + "contracts/crux-O-10-v1.yaml"^^ . + "crux-O-10-v1"^^ . + . + . + "contracts/crux-O-11-v1.yaml"^^ . + "crux-O-11-v1"^^ . + . + . + "contracts/crux-O-12-v1.yaml"^^ . + "crux-O-12-v1"^^ . + . + . + "contracts/crux-O-13-v1.yaml"^^ . + "crux-O-13-v1"^^ . + . + . + "contracts/crux-O-14-v1.yaml"^^ . + "crux-O-14-v1"^^ . + . + . + "contracts/crux-O-15-v1.yaml"^^ . + "crux-O-15-v1"^^ . + . + . + "contracts/crux-O-16-v1.yaml"^^ . + "crux-O-16-v1"^^ . + . + . + "contracts/crux-O-17-v1.yaml"^^ . + "crux-O-17-v1"^^ . + . + . + "contracts/crux-O-18-v1.yaml"^^ . + "crux-O-18-v1"^^ . + . + . + "contracts/crux-O-19-v1.yaml"^^ . + "crux-O-19-v1"^^ . + . + . + "contracts/crux-O-20-v1.yaml"^^ . + "crux-O-20-v1"^^ . + . + . + "contracts/crux-O-21-v1.yaml"^^ . + "crux-O-21-v1"^^ . + . + . + "contracts/crux-O-22-v1.yaml"^^ . + "crux-O-22-v1"^^ . + . + . + "contracts/crux-O-23-v1.yaml"^^ . + "crux-O-23-v1"^^ . + . + . + "contracts/crux-O-24-v1.yaml"^^ . + "crux-O-24-v1"^^ . . . "contracts/crux-competitive-research-ux-v1.yaml"^^ . @@ -7761,7 +7857,7 @@ "contracts/gguf-kquant-element-size-v1.yaml"^^ . "gguf-kquant-element-size-v1"^^ . "gguf-kquant-element-size"^^ . - "1.0.0"^^ . + "1.0.1"^^ . . . "contracts/gguf-prompt-sensitivity-v1.yaml"^^ . @@ -8411,6 +8507,29 @@ . "contracts/entrenar/parity-profiling-system-v1.yaml"^^ . "parity-profiling-system-v1"^^ . + . + . + . + "parity-receipt"^^ . + "contracts/parity-receipt-v1.yaml"^^ . + "parity-receipt-v1"^^ . + "pattern"^^ . + "parity-receipt-v1"^^ . + "superseded"^^ . + "1.0.0"^^ . + . + . + . + . + . + . + "parity-receipt"^^ . + "contracts/parity-receipt-v2.yaml"^^ . + "parity-receipt-v2"^^ . + "pattern"^^ . + "parity-receipt-v2"^^ . + "active"^^ . + "2.0.0"^^ . . . "contracts/bashrs/parser-soundness-v1.yaml"^^ . @@ -8778,6 +8897,16 @@ "reduce-lr-plateau-v1"^^ . "reduce-lr-plateau-patience-strictly-greater"^^ . "1.0.0"^^ . + . + . + "evidence/verbs/refusals.json"^^ . + "json"^^ . + "contracts/refusal-receipt-v1.yaml"^^ . + "refusal-receipt-v1"^^ . + "pattern"^^ . + "refusal-receipt"^^ . + "active"^^ . + "1.0.0"^^ . . . "contracts/pacha/registry-integrity-v1.yaml"^^ . @@ -9409,6 +9538,113 @@ . "false"^^ . "d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785"^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "self"^^ . + "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-08"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-08"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.66.0"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "noah-Lambda-Vector"^^ . + "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-06"^^ . + "noah-Lambda-Vector"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-06"^^ . + "noah-Lambda-Vector"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + "0.65.2"^^ . + "cuda"^^ . + . + "evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json"^^ . + "2026-09-09"^^ . + "gx10-a5b5"^^ . + "true"^^ . + "evidence/parity/thresholds.yaml"^^ . + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp."^^ . + "The exact minute of the run is not recorded; see provenance.generated_at_basis."^^ . + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)."^^ . + . + . . "cpu=ok"^^ . "cuda=ok"^^ . @@ -9825,6 +10061,16 @@ . "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8"^^ . "0.68.2"^^ . + . + . + "The refusal bucket of the verb-set audit (#3597). Every entry is shaped by contracts/refusal-receipt-v1.yaml: a verb from the 111-verb registry, one reason line, the exit code a caller sees, and the release that removes the refusal or a declared sentinel saying none will. #3597 owns WHICH verbs land here; this file is seeded with the one refusal already measured so the shape has a real focus node rather than passing vacuously over an empty list."^^ . + . + "apr-refusal-ledger/v1"^^ . + . + "8"^^ . + "apr bench refuses qwen35 outright: the subcommand routes through the dense QuantizedGGUFTransformer loader, which does not carry the Gated-DeltaNet hybrid, so the benchmark verb cannot measure the architecture the last release shipped."^^ . + "unscheduled"^^ . + "bench"^^ . . . "doc"^^ . diff --git a/contracts/crux-O-01-v1.yaml b/contracts/crux-O-01-v1.yaml new file mode 100644 index 0000000000..e317f539e5 --- /dev/null +++ b/contracts/crux-O-01-v1.yaml @@ -0,0 +1,94 @@ +# CRUX-O-01 — One-call tabular AutoML: fit(label) -> predict on a CSV +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3371, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-01 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3371 + description: > + One-call tabular AutoML: fit(label) -> predict on a CSV. Competitor verb: TabularPredictor(label="class").fit("train.csv"); predictor.predict("test.csv"). Gap measured in aprender at eb262f8eb: No predictor-level AutoML entry point exists. crates/aprender-core/src/automl/ is a hyperparameter TUNER (AutoTuner, TPE, GridSearch, RandomSearch, DESearch, TimeBudget) that tunes ONE estimator the caller already chose; nothing takes a labelled table and returns a fitted model. `apr train` is causal-LM pre-training only (crates/apr-cli/src/commands/train.rs:1-5) and `apr finetune --task classify` is text classification. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3371' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + fit_on_a_labelled_csv_returns_a_predictor_whose_predictions_: + formula: | + on the iris fixture (crates/aprender-core/src/datasets/iris.csv) accuracy >= 0.90 AND the majority-class baseline is asserted at 0.333 in the same test + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline" + the_same_call_works_for_a_regression_label_without_a_problem: + formula: | + a numeric label column yields a regressor whose R^2 on a held-out split exceeds 0.5 AND the mean-predictor baseline is asserted at ~0.0 + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the same call works for a regression label without a problem_type argument" + apr_automl_fit_is_reachable_from_the_cli: + formula: | + `apr automl fit --label class train.csv --out model.apr` exits 0 and writes a loadable artifact; a missing --label exits 2 with a message naming the flag + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "apr automl fit is reachable from the CLI" + +falsification_tests: +- id: FALSIFY-CRUX-O-01-001 + rule: "fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline" + prediction: "on the iris fixture (crates/aprender-core/src/datasets/iris.csv) accuracy >= 0.90 AND the majority-class baseline is asserted at 0.333 in the same test" + test: >- + LIVE-PENDING - fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline. No test surface exists today because the capability is unimplemented: One-call tabular AutoML: fit(label) -> predict on a CSV (aprender#3371, CRUX-O-01). PROMOTE by authoring a test named fit_beats_majority_baseline_on_iris in module `automl/predictor/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline' is violated — the autogluon parity claim for CRUX-O-01 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-01-002 + rule: "the same call works for a regression label without a problem_type argument" + prediction: "a numeric label column yields a regressor whose R^2 on a held-out split exceeds 0.5 AND the mean-predictor baseline is asserted at ~0.0" + test: >- + LIVE-PENDING - the same call works for a regression label without a problem_type argument. No test surface exists today because the capability is unimplemented: One-call tabular AutoML: fit(label) -> predict on a CSV (aprender#3371, CRUX-O-01). PROMOTE by authoring a test named fit_infers_regression_from_numeric_label in module `automl/predictor/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the same call works for a regression label without a problem_type argument' is violated — the autogluon parity claim for CRUX-O-01 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-01-003 + rule: "apr automl fit is reachable from the CLI" + prediction: "`apr automl fit --label class train.csv --out model.apr` exits 0 and writes a loadable artifact; a missing --label exits 2 with a message naming the flag" + test: >- + LIVE-PENDING - apr automl fit is reachable from the CLI. No test surface exists today because the capability is unimplemented: One-call tabular AutoML: fit(label) -> predict on a CSV (aprender#3371, CRUX-O-01). PROMOTE by authoring a test named cli_automl_fit_roundtrip in module `commands/automl_tests` of crate `apr-cli`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'apr automl fit is reachable from the CLI' is violated — the autogluon parity claim for CRUX-O-01 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline" +- type: invariant + property: "the same call works for a regression label without a problem_type argument" +- type: invariant + property: "apr automl fit is reachable from the CLI" + +kani_harnesses: +- id: KH-CRUX-O-01-001 + obligation: fit_on_a_labelled_csv_returns_a_predictor_whose_predictions_ + property: fit_on_a_labelled_csv_returns_a_predictor_whose_predictions__holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-01-002 + obligation: the_same_call_works_for_a_regression_label_without_a_problem + property: the_same_call_works_for_a_regression_label_without_a_problem_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-01-003 + obligation: apr_automl_fit_is_reachable_from_the_cli + property: apr_automl_fit_is_reachable_from_the_cli_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-02-v1.yaml b/contracts/crux-O-02-v1.yaml new file mode 100644 index 0000000000..efe32eda60 --- /dev/null +++ b/contracts/crux-O-02-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-02 — Problem-type inference: binary / multiclass / regression / quantile from the label column +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3372, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-02 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3372 + description: > + Problem-type inference: binary / multiclass / regression / quantile from the label column. Competitor verb: predictor.problem_type (inferred in fit unless problem_type= given). Gap measured in aprender at eb262f8eb: No function infers a task from a label column. Estimators are chosen by type name (LogisticRegression vs LinearRegression); DataFrame in crates/aprender-core/src/data/mod.rs carries ColumnStats but no label-kind classifier. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3372' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + two_unique_label_values_infer_binary__3___n_small_cardinalit: + formula: | + a table of (label column, expected kind) fixtures including the ambiguous cases {0,1} as int, {0.0,1.0} as float, and 30 unique floats over 1000 rows all classify as documented + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression" + the_override_wins_and_a_contradictory_override_is_rejected: + formula: | + problem_type=regression on a string label column is an Err naming the column, not a silent cast + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the override wins and a contradictory override is rejected" + +falsification_tests: +- id: FALSIFY-CRUX-O-02-001 + rule: "two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression" + prediction: "a table of (label column, expected kind) fixtures including the ambiguous cases {0,1} as int, {0.0,1.0} as float, and 30 unique floats over 1000 rows all classify as documented" + test: >- + LIVE-PENDING - two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression. No test surface exists today because the capability is unimplemented: Problem-type inference: binary / multiclass / regression / quantile from the label column (aprender#3372, CRUX-O-02). PROMOTE by authoring a test named label_kind_table in module `automl/problem_type/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression' is violated — the autogluon parity claim for CRUX-O-02 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-02-002 + rule: "the override wins and a contradictory override is rejected" + prediction: "problem_type=regression on a string label column is an Err naming the column, not a silent cast" + test: >- + LIVE-PENDING - the override wins and a contradictory override is rejected. No test surface exists today because the capability is unimplemented: Problem-type inference: binary / multiclass / regression / quantile from the label column (aprender#3372, CRUX-O-02). PROMOTE by authoring a test named override_contradiction_is_error in module `automl/problem_type/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the override wins and a contradictory override is rejected' is violated — the autogluon parity claim for CRUX-O-02 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression" +- type: invariant + property: "the override wins and a contradictory override is rejected" + +kani_harnesses: +- id: KH-CRUX-O-02-001 + obligation: two_unique_label_values_infer_binary__3___n_small_cardinalit + property: two_unique_label_values_infer_binary__3___n_small_cardinalit_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-02-002 + obligation: the_override_wins_and_a_contradictory_override_is_rejected + property: the_override_wins_and_a_contradictory_override_is_rejected_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-03-v1.yaml b/contracts/crux-O-03-v1.yaml new file mode 100644 index 0000000000..13dc0c8df0 --- /dev/null +++ b/contracts/crux-O-03-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-03 — Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3373, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-03 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3373 + description: > + Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget. Competitor verb: fit(..., presets="best_quality") # tabular/src/autogluon/tabular/configs/presets_configs.py. Gap measured in aprender at eb262f8eb: No preset vocabulary. AutoTuner takes a SearchSpace the caller hand-builds (crates/aprender-core/src/automl/params.rs); there is no named bundle of {models, bagging, stacking, time_limit}. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3373' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + every_preset_name_resolves_to_a_portfolio_and_unknown_names_: + formula: | + the five quality presets each yield a non-empty ordered model list; `presets="bestest"` is an Err whose message contains all five valid names + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "every preset name resolves to a portfolio and unknown names are rejected with the valid list" + presets_are_ordered__a_higher_preset_never_fits_fewer_model_: + formula: | + for medium < good < high < best the family count is monotone non-decreasing, asserted pairwise + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "presets are ordered: a higher preset never fits FEWER model families than the one below it" + +falsification_tests: +- id: FALSIFY-CRUX-O-03-001 + rule: "every preset name resolves to a portfolio and unknown names are rejected with the valid list" + prediction: "the five quality presets each yield a non-empty ordered model list; `presets=\"bestest\"` is an Err whose message contains all five valid names" + test: >- + LIVE-PENDING - every preset name resolves to a portfolio and unknown names are rejected with the valid list. No test surface exists today because the capability is unimplemented: Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget (aprender#3373, CRUX-O-03). PROMOTE by authoring a test named preset_table_resolves in module `automl/presets/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'every preset name resolves to a portfolio and unknown names are rejected with the valid list' is violated — the autogluon parity claim for CRUX-O-03 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-03-002 + rule: "presets are ordered: a higher preset never fits FEWER model families than the one below it" + prediction: "for medium < good < high < best the family count is monotone non-decreasing, asserted pairwise" + test: >- + LIVE-PENDING - presets are ordered: a higher preset never fits FEWER model families than the one below it. No test surface exists today because the capability is unimplemented: Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget (aprender#3373, CRUX-O-03). PROMOTE by authoring a test named preset_monotone_families in module `automl/presets/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'presets are ordered: a higher preset never fits FEWER model families than the one below it' is violated — the autogluon parity claim for CRUX-O-03 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "every preset name resolves to a portfolio and unknown names are rejected with the valid list" +- type: invariant + property: "presets are ordered: a higher preset never fits FEWER model families than the one below it" + +kani_harnesses: +- id: KH-CRUX-O-03-001 + obligation: every_preset_name_resolves_to_a_portfolio_and_unknown_names_ + property: every_preset_name_resolves_to_a_portfolio_and_unknown_names__holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-03-002 + obligation: presets_are_ordered__a_higher_preset_never_fits_fewer_model_ + property: presets_are_ordered__a_higher_preset_never_fits_fewer_model__holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-04-v1.yaml b/contracts/crux-O-04-v1.yaml new file mode 100644 index 0000000000..4b28ca9ccc --- /dev/null +++ b/contracts/crux-O-04-v1.yaml @@ -0,0 +1,94 @@ +# CRUX-O-04 — Leaderboard: per-model validation/test score, fit time, predict time and stack level +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3374, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-04 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3374 + description: > + Leaderboard: per-model validation/test score, fit time, predict time and stack level. Competitor verb: predictor.leaderboard(test_data, extra_info=True). Gap measured in aprender at eb262f8eb: No leaderboard type. GridSearchCVResult in model_selection/ ranks parameter settings of one estimator; TuneResult in automl/tuner.rs is a single best trial. Nothing tabulates several fitted models with timings. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3374' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_leaderboard_is_sorted_by_validation_score_descending_and: + formula: | + columns == [model, score_val, score_test?, pred_time_val, fit_time, stack_level, fit_order] and score_val is non-increasing row to row + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the leaderboard is sorted by validation score descending and its columns are fixed" + timings_are_measured__not_defaulted: + formula: | + every fit_time and pred_time_val is > 0 after a real fit; a leaderboard built with no fit has zero rows, not zero timings + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "timings are measured, not defaulted" + apr_automl_leaderboard_prints_the_same_table_from_a_saved_ar: + formula: | + `apr automl leaderboard model.apr --json` emits the rows byte-equal to the in-process leaderboard + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "apr automl leaderboard prints the same table from a saved artifact" + +falsification_tests: +- id: FALSIFY-CRUX-O-04-001 + rule: "the leaderboard is sorted by validation score descending and its columns are fixed" + prediction: "columns == [model, score_val, score_test?, pred_time_val, fit_time, stack_level, fit_order] and score_val is non-increasing row to row" + test: >- + LIVE-PENDING - the leaderboard is sorted by validation score descending and its columns are fixed. No test surface exists today because the capability is unimplemented: Leaderboard: per-model validation/test score, fit time, predict time and stack level (aprender#3374, CRUX-O-04). PROMOTE by authoring a test named leaderboard_sorted_and_typed in module `automl/leaderboard/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the leaderboard is sorted by validation score descending and its columns are fixed' is violated — the autogluon parity claim for CRUX-O-04 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-04-002 + rule: "timings are measured, not defaulted" + prediction: "every fit_time and pred_time_val is > 0 after a real fit; a leaderboard built with no fit has zero rows, not zero timings" + test: >- + LIVE-PENDING - timings are measured, not defaulted. No test surface exists today because the capability is unimplemented: Leaderboard: per-model validation/test score, fit time, predict time and stack level (aprender#3374, CRUX-O-04). PROMOTE by authoring a test named leaderboard_timings_are_measured in module `automl/leaderboard/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'timings are measured, not defaulted' is violated — the autogluon parity claim for CRUX-O-04 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-04-003 + rule: "apr automl leaderboard prints the same table from a saved artifact" + prediction: "`apr automl leaderboard model.apr --json` emits the rows byte-equal to the in-process leaderboard" + test: >- + LIVE-PENDING - apr automl leaderboard prints the same table from a saved artifact. No test surface exists today because the capability is unimplemented: Leaderboard: per-model validation/test score, fit time, predict time and stack level (aprender#3374, CRUX-O-04). PROMOTE by authoring a test named cli_leaderboard_matches_library in module `commands/automl_tests` of crate `apr-cli`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'apr automl leaderboard prints the same table from a saved artifact' is violated — the autogluon parity claim for CRUX-O-04 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the leaderboard is sorted by validation score descending and its columns are fixed" +- type: invariant + property: "timings are measured, not defaulted" +- type: invariant + property: "apr automl leaderboard prints the same table from a saved artifact" + +kani_harnesses: +- id: KH-CRUX-O-04-001 + obligation: the_leaderboard_is_sorted_by_validation_score_descending_and + property: the_leaderboard_is_sorted_by_validation_score_descending_and_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-04-002 + obligation: timings_are_measured__not_defaulted + property: timings_are_measured__not_defaulted_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-04-003 + obligation: apr_automl_leaderboard_prints_the_same_table_from_a_saved_ar + property: apr_automl_leaderboard_prints_the_same_table_from_a_saved_ar_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-05-v1.yaml b/contracts/crux-O-05-v1.yaml new file mode 100644 index 0000000000..e490ee142c --- /dev/null +++ b/contracts/crux-O-05-v1.yaml @@ -0,0 +1,94 @@ +# CRUX-O-05 — Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3375, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-05 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: partial + github_issue: 3375 + description: > + Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate). Competitor verb: AutoMLPipelineFeatureGenerator # features/src/autogluon/features/generators/auto_ml_pipeline.py. Gap measured in aprender at eb262f8eb: Encoders exist (LabelEncoder, OneHotEncoder, OrdinalEncoder, StandardScaler, PolynomialFeatures in crates/aprender-core/src/preprocessing/) but every one is applied by hand to a column the caller already typed. There is no pass that reads a raw DataFrame, infers each column's kind, and emits a fitted transform. Datetime expansion and text n-gram features do not exist. TfidfVectorizer exists in text/ but is not wired to a tabular pipeline. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3375' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + column_kinds_are_inferred_from_raw_values: + formula: | + a fixture CSV with int, float, low-cardinality string, high-cardinality string, ISO datetime and free-text columns is typed as {numeric, numeric, categorical, text, datetime, text} exactly + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "column kinds are inferred from raw values" + constant_and_duplicate_columns_are_dropped_and_the_drop_is_r: + formula: | + a column with one unique value and an exact duplicate of another column are both absent from the transformed output AND named in the fit report + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "constant and duplicate columns are dropped and the drop is reported" + the_fitted_pipeline_is_deterministic_under_transform: + formula: | + transform(train) then transform(train) are byte-identical and transform(test) never sees a category unseen at fit as anything but the reserved unknown code + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the fitted pipeline is deterministic under transform" + +falsification_tests: +- id: FALSIFY-CRUX-O-05-001 + rule: "column kinds are inferred from raw values" + prediction: "a fixture CSV with int, float, low-cardinality string, high-cardinality string, ISO datetime and free-text columns is typed as {numeric, numeric, categorical, text, datetime, text} exactly" + test: >- + LIVE-PENDING - column kinds are inferred from raw values. No test surface exists today because the capability is partial: Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) (aprender#3375, CRUX-O-05). PROMOTE by authoring a test named infer_feature_kinds_fixture in module `automl/features/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'column kinds are inferred from raw values' is violated — the autogluon parity claim for CRUX-O-05 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-05-002 + rule: "constant and duplicate columns are dropped and the drop is reported" + prediction: "a column with one unique value and an exact duplicate of another column are both absent from the transformed output AND named in the fit report" + test: >- + LIVE-PENDING - constant and duplicate columns are dropped and the drop is reported. No test surface exists today because the capability is partial: Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) (aprender#3375, CRUX-O-05). PROMOTE by authoring a test named drop_unique_and_duplicate_reported in module `automl/features/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'constant and duplicate columns are dropped and the drop is reported' is violated — the autogluon parity claim for CRUX-O-05 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-05-003 + rule: "the fitted pipeline is deterministic under transform" + prediction: "transform(train) then transform(train) are byte-identical and transform(test) never sees a category unseen at fit as anything but the reserved unknown code" + test: >- + LIVE-PENDING - the fitted pipeline is deterministic under transform. No test surface exists today because the capability is partial: Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) (aprender#3375, CRUX-O-05). PROMOTE by authoring a test named pipeline_transform_deterministic in module `automl/features/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the fitted pipeline is deterministic under transform' is violated — the autogluon parity claim for CRUX-O-05 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "column kinds are inferred from raw values" +- type: invariant + property: "constant and duplicate columns are dropped and the drop is reported" +- type: invariant + property: "the fitted pipeline is deterministic under transform" + +kani_harnesses: +- id: KH-CRUX-O-05-001 + obligation: column_kinds_are_inferred_from_raw_values + property: column_kinds_are_inferred_from_raw_values_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-05-002 + obligation: constant_and_duplicate_columns_are_dropped_and_the_drop_is_r + property: constant_and_duplicate_columns_are_dropped_and_the_drop_is_r_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-05-003 + obligation: the_fitted_pipeline_is_deterministic_under_transform + property: the_fitted_pipeline_is_deterministic_under_transform_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-06-v1.yaml b/contracts/crux-O-06-v1.yaml new file mode 100644 index 0000000000..750fbebeeb --- /dev/null +++ b/contracts/crux-O-06-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-06 — K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3376, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-06 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3376 + description: > + K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof). Competitor verb: fit(..., num_bag_folds=8); predictor.predict_proba_oof(). Gap measured in aprender at eb262f8eb: KFold and StratifiedKFold exist (model_selection/) and cross_validate scores them, but no wrapper trains one child per fold, keeps all children, averages them at predict time and exposes the out-of-fold matrix. RandomForest bags trees internally and is not reusable for other estimators. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3376' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + oof_predictions_cover_every_training_row_exactly_once: + formula: | + for n rows and k folds the OOF matrix has n rows, no NaN, and each row was predicted by the one child that did not see it (asserted through a fold-id trace) + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "OOF predictions cover every training row exactly once" + bagged_prediction_is_the_mean_of_the_children: + formula: | + predict_proba of the bag equals the elementwise mean of the k children's predict_proba within 1e-12 + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "bagged prediction is the mean of the children" + +falsification_tests: +- id: FALSIFY-CRUX-O-06-001 + rule: "OOF predictions cover every training row exactly once" + prediction: "for n rows and k folds the OOF matrix has n rows, no NaN, and each row was predicted by the one child that did not see it (asserted through a fold-id trace)" + test: >- + LIVE-PENDING - OOF predictions cover every training row exactly once. No test surface exists today because the capability is unimplemented: K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) (aprender#3376, CRUX-O-06). PROMOTE by authoring a test named oof_covers_each_row_once in module `automl/bagging/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'OOF predictions cover every training row exactly once' is violated — the autogluon parity claim for CRUX-O-06 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-06-002 + rule: "bagged prediction is the mean of the children" + prediction: "predict_proba of the bag equals the elementwise mean of the k children's predict_proba within 1e-12" + test: >- + LIVE-PENDING - bagged prediction is the mean of the children. No test surface exists today because the capability is unimplemented: K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) (aprender#3376, CRUX-O-06). PROMOTE by authoring a test named bag_predict_is_child_mean in module `automl/bagging/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'bagged prediction is the mean of the children' is violated — the autogluon parity claim for CRUX-O-06 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "OOF predictions cover every training row exactly once" +- type: invariant + property: "bagged prediction is the mean of the children" + +kani_harnesses: +- id: KH-CRUX-O-06-001 + obligation: oof_predictions_cover_every_training_row_exactly_once + property: oof_predictions_cover_every_training_row_exactly_once_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-06-002 + obligation: bagged_prediction_is_the_mean_of_the_children + property: bagged_prediction_is_the_mean_of_the_children_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-07-v1.yaml b/contracts/crux-O-07-v1.yaml new file mode 100644 index 0000000000..e76ba2a33c --- /dev/null +++ b/contracts/crux-O-07-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-07 — Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3377, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-07 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3377 + description: > + Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking). Competitor verb: fit(..., num_stack_levels=1, dynamic_stacking="auto"). Gap measured in aprender at eb262f8eb: No stacking. crates/aprender-core/src/stack/ is a deployment-health module (StackHealth, InferenceConfig), not a model stacker. ensemble/ holds MixtureOfExperts gating only. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3377' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + level_2_features_are_oof__never_in_sample: + formula: | + a mutation that feeds in-sample level-1 predictions to level 2 is detected by the leak test: the L2 holdout score on a pure-noise label rises above chance (asserted RED) while the OOF path stays at chance + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "level-2 features are OOF, never in-sample" + dynamic_stacking_falls_back_when_stacking_hurts: + formula: | + on a fixture where L2 holdout score < L1 holdout score, the final model is the L1 ensemble and the decision is recorded in the fit summary + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "dynamic stacking falls back when stacking hurts" + +falsification_tests: +- id: FALSIFY-CRUX-O-07-001 + rule: "level-2 features are OOF, never in-sample" + prediction: "a mutation that feeds in-sample level-1 predictions to level 2 is detected by the leak test: the L2 holdout score on a pure-noise label rises above chance (asserted RED) while the OOF path stays at chance" + test: >- + LIVE-PENDING - level-2 features are OOF, never in-sample. No test surface exists today because the capability is unimplemented: Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) (aprender#3377, CRUX-O-07). PROMOTE by authoring a test named stack_uses_oof_not_insample in module `automl/stacking/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'level-2 features are OOF, never in-sample' is violated — the autogluon parity claim for CRUX-O-07 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-07-002 + rule: "dynamic stacking falls back when stacking hurts" + prediction: "on a fixture where L2 holdout score < L1 holdout score, the final model is the L1 ensemble and the decision is recorded in the fit summary" + test: >- + LIVE-PENDING - dynamic stacking falls back when stacking hurts. No test surface exists today because the capability is unimplemented: Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) (aprender#3377, CRUX-O-07). PROMOTE by authoring a test named dynamic_stacking_fallback_recorded in module `automl/stacking/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'dynamic stacking falls back when stacking hurts' is violated — the autogluon parity claim for CRUX-O-07 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "level-2 features are OOF, never in-sample" +- type: invariant + property: "dynamic stacking falls back when stacking hurts" + +kani_harnesses: +- id: KH-CRUX-O-07-001 + obligation: level_2_features_are_oof__never_in_sample + property: level_2_features_are_oof__never_in_sample_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-07-002 + obligation: dynamic_stacking_falls_back_when_stacking_hurts + property: dynamic_stacking_falls_back_when_stacking_hurts_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-08-v1.yaml b/contracts/crux-O-08-v1.yaml new file mode 100644 index 0000000000..cc1700640e --- /dev/null +++ b/contracts/crux-O-08-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-08 — Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3378, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-08 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3378 + description: > + Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection). Competitor verb: fit_weighted_ensemble=True (default) # core/src/autogluon/core/models/greedy_ensemble/ensemble_selection.py. Gap measured in aprender at eb262f8eb: No ensemble-selection algorithm. MixtureOfExperts learns a gating network (ensemble/moe.rs); nothing performs the forward greedy selection with replacement over base-model validation predictions that yields non-negative weights summing to 1. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3378' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + weights_are_a_probability_vector_and_the_ensemble_never_scor: + formula: | + sum(w)=1, all w>=0, and validation metric(ensemble) >= max over members within 1e-9, on 3 fixtures + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "weights are a probability vector and the ensemble never scores below its best member" + selection_is_greedy_with_replacement_and_reproducible: + formula: | + with ensemble_size=25 the weight of a member equals its selection count / 25; two runs on the same inputs produce identical weights + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "selection is greedy with replacement and reproducible" + +falsification_tests: +- id: FALSIFY-CRUX-O-08-001 + rule: "weights are a probability vector and the ensemble never scores below its best member" + prediction: "sum(w)=1, all w>=0, and validation metric(ensemble) >= max over members within 1e-9, on 3 fixtures" + test: >- + LIVE-PENDING - weights are a probability vector and the ensemble never scores below its best member. No test surface exists today because the capability is unimplemented: Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) (aprender#3378, CRUX-O-08). PROMOTE by authoring a test named weights_simplex_and_no_regression in module `automl/ensemble_selection/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'weights are a probability vector and the ensemble never scores below its best member' is violated — the autogluon parity claim for CRUX-O-08 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-08-002 + rule: "selection is greedy with replacement and reproducible" + prediction: "with ensemble_size=25 the weight of a member equals its selection count / 25; two runs on the same inputs produce identical weights" + test: >- + LIVE-PENDING - selection is greedy with replacement and reproducible. No test surface exists today because the capability is unimplemented: Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) (aprender#3378, CRUX-O-08). PROMOTE by authoring a test named greedy_with_replacement_counts in module `automl/ensemble_selection/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'selection is greedy with replacement and reproducible' is violated — the autogluon parity claim for CRUX-O-08 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "weights are a probability vector and the ensemble never scores below its best member" +- type: invariant + property: "selection is greedy with replacement and reproducible" + +kani_harnesses: +- id: KH-CRUX-O-08-001 + obligation: weights_are_a_probability_vector_and_the_ensemble_never_scor + property: weights_are_a_probability_vector_and_the_ensemble_never_scor_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-08-002 + obligation: selection_is_greedy_with_replacement_and_reproducible + property: selection_is_greedy_with_replacement_and_reproducible_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-09-v1.yaml b/contracts/crux-O-09-v1.yaml new file mode 100644 index 0000000000..5bfadb368f --- /dev/null +++ b/contracts/crux-O-09-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-09 — Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3379, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-09 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: partial + github_issue: 3379 + description: > + Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share. Competitor verb: fit(..., time_limit=3600). Gap measured in aprender at eb262f8eb: TimeBudget and EarlyStopping exist in automl/tuner.rs but budget ONE tuner. There is no allocation of a global limit across an ordered portfolio, no per-model time share, and no 'skip the rest' when the budget is exhausted. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3379' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_wall_clock_of_fit_never_exceeds_time_limit_by_more_than_: + formula: | + with time_limit=5s on a portfolio that would take >60s unconstrained, elapsed <= 5s + 1s AND at least one model was skipped with reason=budget in the summary + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the wall-clock of fit never exceeds time_limit by more than the tolerance" + the_budget_is_redistributed_when_a_model_finishes_early: + formula: | + a model that uses 10% of its share returns the remainder to the pool; the next model's share is asserted larger than the naive equal split + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the budget is redistributed when a model finishes early" + +falsification_tests: +- id: FALSIFY-CRUX-O-09-001 + rule: "the wall-clock of fit never exceeds time_limit by more than the tolerance" + prediction: "with time_limit=5s on a portfolio that would take >60s unconstrained, elapsed <= 5s + 1s AND at least one model was skipped with reason=budget in the summary" + test: >- + LIVE-PENDING - the wall-clock of fit never exceeds time_limit by more than the tolerance. No test surface exists today because the capability is partial: Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share (aprender#3379, CRUX-O-09). PROMOTE by authoring a test named portfolio_respects_global_budget in module `automl/budget/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the wall-clock of fit never exceeds time_limit by more than the tolerance' is violated — the autogluon parity claim for CRUX-O-09 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-09-002 + rule: "the budget is redistributed when a model finishes early" + prediction: "a model that uses 10% of its share returns the remainder to the pool; the next model's share is asserted larger than the naive equal split" + test: >- + LIVE-PENDING - the budget is redistributed when a model finishes early. No test surface exists today because the capability is partial: Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share (aprender#3379, CRUX-O-09). PROMOTE by authoring a test named unused_share_is_redistributed in module `automl/budget/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the budget is redistributed when a model finishes early' is violated — the autogluon parity claim for CRUX-O-09 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the wall-clock of fit never exceeds time_limit by more than the tolerance" +- type: invariant + property: "the budget is redistributed when a model finishes early" + +kani_harnesses: +- id: KH-CRUX-O-09-001 + obligation: the_wall_clock_of_fit_never_exceeds_time_limit_by_more_than_ + property: the_wall_clock_of_fit_never_exceeds_time_limit_by_more_than__holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-09-002 + obligation: the_budget_is_redistributed_when_a_model_finishes_early + property: the_budget_is_redistributed_when_a_model_finishes_early_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-10-v1.yaml b/contracts/crux-O-10-v1.yaml new file mode 100644 index 0000000000..bbabd53a5a --- /dev/null +++ b/contracts/crux-O-10-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-10 — Predictor-level permutation feature importance with p-values and confidence intervals +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3380, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-10 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: partial + github_issue: 3380 + description: > + Predictor-level permutation feature importance with p-values and confidence intervals. Competitor verb: predictor.feature_importance(test_data, num_shuffle_sets=10). Gap measured in aprender at eb262f8eb: PermutationImportance exists in crates/aprender-core/src/interpret/ for a single estimator. Missing: the predictor-level call on raw (pre-pipeline) columns, num_shuffle_sets repeats, stddev / p-value / p99 columns, and subsampling. AutoGluon 1.6 also cut this call's memory 25x (#5645). + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3380' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + importance_is_reported_per_raw_input_column: + formula: | + on a fixture whose datetime column expands to 4 features, the importance table has one row for the datetime column, not four + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "importance is reported per raw input column" + a_pure_noise_column_has_importance_statistically_indistingui: + formula: | + with num_shuffle_sets=10 the noise column's p-value > 0.05 AND the signal column's p-value < 0.01 on the same run + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a pure-noise column has importance statistically indistinguishable from zero" + +falsification_tests: +- id: FALSIFY-CRUX-O-10-001 + rule: "importance is reported per raw input column" + prediction: "on a fixture whose datetime column expands to 4 features, the importance table has one row for the datetime column, not four" + test: >- + LIVE-PENDING - importance is reported per raw input column. No test surface exists today because the capability is partial: Predictor-level permutation feature importance with p-values and confidence intervals (aprender#3380, CRUX-O-10). PROMOTE by authoring a test named importance_on_raw_columns in module `automl/importance/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'importance is reported per raw input column' is violated — the autogluon parity claim for CRUX-O-10 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-10-002 + rule: "a pure-noise column has importance statistically indistinguishable from zero" + prediction: "with num_shuffle_sets=10 the noise column's p-value > 0.05 AND the signal column's p-value < 0.01 on the same run" + test: >- + LIVE-PENDING - a pure-noise column has importance statistically indistinguishable from zero. No test surface exists today because the capability is partial: Predictor-level permutation feature importance with p-values and confidence intervals (aprender#3380, CRUX-O-10). PROMOTE by authoring a test named noise_column_pvalue in module `automl/importance/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a pure-noise column has importance statistically indistinguishable from zero' is violated — the autogluon parity claim for CRUX-O-10 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "importance is reported per raw input column" +- type: invariant + property: "a pure-noise column has importance statistically indistinguishable from zero" + +kani_harnesses: +- id: KH-CRUX-O-10-001 + obligation: importance_is_reported_per_raw_input_column + property: importance_is_reported_per_raw_input_column_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-10-002 + obligation: a_pure_noise_column_has_importance_statistically_indistingui + property: a_pure_noise_column_has_importance_statistically_indistingui_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-11-v1.yaml b/contracts/crux-O-11-v1.yaml new file mode 100644 index 0000000000..2132d3123c --- /dev/null +++ b/contracts/crux-O-11-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-11 — Decision-threshold calibration for binary metrics (calibrate_decision_threshold) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3381, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-11 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: partial + github_issue: 3381 + description: > + Decision-threshold calibration for binary metrics (calibrate_decision_threshold). Competitor verb: fit(..., calibrate_decision_threshold="auto"); predictor.calibrate_decision_threshold(metric="f1"). Gap measured in aprender at eb262f8eb: Probability calibration exists (PlattScaling, IsotonicRegression, TemperatureScaling in calibration.rs) but nothing searches the decision threshold that maximises f1 / balanced_accuracy / mcc on validation data and stores it on the predictor (`decision_threshold`, `set_decision_threshold`). + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3381' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_calibrated_threshold_beats_0_5_on_the_calibration_metric: + formula: | + on an imbalanced fixture (5% positives) f1 at the calibrated threshold > f1 at 0.5 by >= 0.05 absolute + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search" + the_threshold_is_persisted_with_the_model: + formula: | + save then load reproduces predict() bit-identically including the threshold; a mutation that resets the threshold to 0.5 on load turns this RED + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the threshold is persisted with the model" + +falsification_tests: +- id: FALSIFY-CRUX-O-11-001 + rule: "the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search" + prediction: "on an imbalanced fixture (5% positives) f1 at the calibrated threshold > f1 at 0.5 by >= 0.05 absolute" + test: >- + LIVE-PENDING - the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search. No test surface exists today because the capability is partial: Decision-threshold calibration for binary metrics (calibrate_decision_threshold) (aprender#3381, CRUX-O-11). PROMOTE by authoring a test named calibrated_threshold_beats_half in module `automl/threshold/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search' is violated — the autogluon parity claim for CRUX-O-11 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-11-002 + rule: "the threshold is persisted with the model" + prediction: "save then load reproduces predict() bit-identically including the threshold; a mutation that resets the threshold to 0.5 on load turns this RED" + test: >- + LIVE-PENDING - the threshold is persisted with the model. No test surface exists today because the capability is partial: Decision-threshold calibration for binary metrics (calibrate_decision_threshold) (aprender#3381, CRUX-O-11). PROMOTE by authoring a test named threshold_persisted_roundtrip in module `automl/threshold/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the threshold is persisted with the model' is violated — the autogluon parity claim for CRUX-O-11 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search" +- type: invariant + property: "the threshold is persisted with the model" + +kani_harnesses: +- id: KH-CRUX-O-11-001 + obligation: the_calibrated_threshold_beats_0_5_on_the_calibration_metric + property: the_calibrated_threshold_beats_0_5_on_the_calibration_metric_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-11-002 + obligation: the_threshold_is_persisted_with_the_model + property: the_threshold_is_persisted_with_the_model_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-12-v1.yaml b/contracts/crux-O-12-v1.yaml new file mode 100644 index 0000000000..fb7576e04a --- /dev/null +++ b/contracts/crux-O-12-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-12 — refit_full: retrain the selected models on train+validation after model selection +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3382, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-12 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3382 + description: > + refit_full: retrain the selected models on train+validation after model selection. Competitor verb: fit(..., refit_full=True, set_best_to_refit_full=True); predictor.refit_full(). Gap measured in aprender at eb262f8eb: No refit-on-full-data step. cross_validate and grid_search return scores; the model returned is the one fitted on a fold, not on all rows. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3382' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_refit_model_saw_every_row: + formula: | + the refit estimator's training-row count equals n_train + n_val, asserted through the fitted row count it reports + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the refit model saw every row" + refit_keeps_the_selected_hyperparameters_and_drops_the_bag_c: + formula: | + the artifact after refit has one child per selected model and its hyperparameters are byte-equal to the pre-refit winner's + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "refit keeps the selected hyperparameters and drops the bag children" + +falsification_tests: +- id: FALSIFY-CRUX-O-12-001 + rule: "the refit model saw every row" + prediction: "the refit estimator's training-row count equals n_train + n_val, asserted through the fitted row count it reports" + test: >- + LIVE-PENDING - the refit model saw every row. No test surface exists today because the capability is unimplemented: refit_full: retrain the selected models on train+validation after model selection (aprender#3382, CRUX-O-12). PROMOTE by authoring a test named refit_sees_all_rows in module `automl/refit/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the refit model saw every row' is violated — the autogluon parity claim for CRUX-O-12 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-12-002 + rule: "refit keeps the selected hyperparameters and drops the bag children" + prediction: "the artifact after refit has one child per selected model and its hyperparameters are byte-equal to the pre-refit winner's" + test: >- + LIVE-PENDING - refit keeps the selected hyperparameters and drops the bag children. No test surface exists today because the capability is unimplemented: refit_full: retrain the selected models on train+validation after model selection (aprender#3382, CRUX-O-12). PROMOTE by authoring a test named refit_keeps_params_drops_children in module `automl/refit/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'refit keeps the selected hyperparameters and drops the bag children' is violated — the autogluon parity claim for CRUX-O-12 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the refit model saw every row" +- type: invariant + property: "refit keeps the selected hyperparameters and drops the bag children" + +kani_harnesses: +- id: KH-CRUX-O-12-001 + obligation: the_refit_model_saw_every_row + property: the_refit_model_saw_every_row_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-12-002 + obligation: refit_keeps_the_selected_hyperparameters_and_drops_the_bag_c + property: refit_keeps_the_selected_hyperparameters_and_drops_the_bag_c_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-13-v1.yaml b/contracts/crux-O-13-v1.yaml new file mode 100644 index 0000000000..a9ea70615f --- /dev/null +++ b/contracts/crux-O-13-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-13 — Model distillation: compress the ensemble into one fast student +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3383, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-13 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3383 + description: > + Model distillation: compress the ensemble into one fast student. Competitor verb: predictor.distill(time_limit=..., augment_method="spunge"). Gap measured in aprender at eb262f8eb: Distillation exists only for LLMs in the training crate (knowledge distillation of transformers); nothing trains a single tabular student on the teacher ensemble's soft labels with data augmentation. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3383' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_student_is_faster_and_within_tolerance_of_the_teacher: + formula: | + student pred_time < 0.25 x teacher pred_time AND student validation score >= teacher score - 0.02 on a fixture + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the student is faster and within tolerance of the teacher" + augmentation_produces_rows_the_training_set_does_not_contain: + formula: | + with augment_method=spunge the student's training set has > n_train rows and the added rows are not row-equal to any original + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "augmentation produces rows the training set does not contain" + +falsification_tests: +- id: FALSIFY-CRUX-O-13-001 + rule: "the student is faster and within tolerance of the teacher" + prediction: "student pred_time < 0.25 x teacher pred_time AND student validation score >= teacher score - 0.02 on a fixture" + test: >- + LIVE-PENDING - the student is faster and within tolerance of the teacher. No test surface exists today because the capability is unimplemented: Model distillation: compress the ensemble into one fast student (aprender#3383, CRUX-O-13). PROMOTE by authoring a test named student_faster_within_tolerance in module `automl/distill/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the student is faster and within tolerance of the teacher' is violated — the autogluon parity claim for CRUX-O-13 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-13-002 + rule: "augmentation produces rows the training set does not contain" + prediction: "with augment_method=spunge the student's training set has > n_train rows and the added rows are not row-equal to any original" + test: >- + LIVE-PENDING - augmentation produces rows the training set does not contain. No test surface exists today because the capability is unimplemented: Model distillation: compress the ensemble into one fast student (aprender#3383, CRUX-O-13). PROMOTE by authoring a test named augmented_rows_are_novel in module `automl/distill/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'augmentation produces rows the training set does not contain' is violated — the autogluon parity claim for CRUX-O-13 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the student is faster and within tolerance of the teacher" +- type: invariant + property: "augmentation produces rows the training set does not contain" + +kani_harnesses: +- id: KH-CRUX-O-13-001 + obligation: the_student_is_faster_and_within_tolerance_of_the_teacher + property: the_student_is_faster_and_within_tolerance_of_the_teacher_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-13-002 + obligation: augmentation_produces_rows_the_training_set_does_not_contain + property: augmentation_produces_rows_the_training_set_does_not_contain_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-14-v1.yaml b/contracts/crux-O-14-v1.yaml new file mode 100644 index 0000000000..a05ea6978c --- /dev/null +++ b/contracts/crux-O-14-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-14 — Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3384, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-14 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3384 + description: > + Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file. Competitor verb: predictor.clone_for_deployment(path); predictor.persist(). Gap measured in aprender at eb262f8eb: Single estimators serialize to .apr (bundle/, serialization/). No artifact holds a fitted feature pipeline (O-05) plus several models plus ensemble weights plus a decision threshold and loads them as one predictor. Nothing prunes non-selected models from disk. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3384' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + a_deployment_clone_is_smaller_and_predicts_identically: + formula: | + clone_for_deployment artifact bytes < 0.5 x full artifact bytes AND predict() on the clone is bit-identical to predict() on the original for the test fixture + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a deployment clone is smaller and predicts identically" + the_artifact_is_self_describing_and_refuses_a_schema_drift: + formula: | + load() on an artifact whose feature schema disagrees with the input columns is an Err naming the first mismatched column, not a wrong prediction + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the artifact is self-describing and refuses a schema drift" + +falsification_tests: +- id: FALSIFY-CRUX-O-14-001 + rule: "a deployment clone is smaller and predicts identically" + prediction: "clone_for_deployment artifact bytes < 0.5 x full artifact bytes AND predict() on the clone is bit-identical to predict() on the original for the test fixture" + test: >- + LIVE-PENDING - a deployment clone is smaller and predicts identically. No test surface exists today because the capability is unimplemented: Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file (aprender#3384, CRUX-O-14). PROMOTE by authoring a test named deployment_clone_smaller_identical in module `automl/artifact/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a deployment clone is smaller and predicts identically' is violated — the autogluon parity claim for CRUX-O-14 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-14-002 + rule: "the artifact is self-describing and refuses a schema drift" + prediction: "load() on an artifact whose feature schema disagrees with the input columns is an Err naming the first mismatched column, not a wrong prediction" + test: >- + LIVE-PENDING - the artifact is self-describing and refuses a schema drift. No test surface exists today because the capability is unimplemented: Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file (aprender#3384, CRUX-O-14). PROMOTE by authoring a test named artifact_refuses_schema_drift in module `automl/artifact/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the artifact is self-describing and refuses a schema drift' is violated — the autogluon parity claim for CRUX-O-14 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "a deployment clone is smaller and predicts identically" +- type: invariant + property: "the artifact is self-describing and refuses a schema drift" + +kani_harnesses: +- id: KH-CRUX-O-14-001 + obligation: a_deployment_clone_is_smaller_and_predicts_identically + property: a_deployment_clone_is_smaller_and_predicts_identically_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-14-002 + obligation: the_artifact_is_self_describing_and_refuses_a_schema_drift + property: the_artifact_is_self_describing_and_refuses_a_schema_drift_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-15-v1.yaml b/contracts/crux-O-15-v1.yaml new file mode 100644 index 0000000000..b70a81b5d8 --- /dev/null +++ b/contracts/crux-O-15-v1.yaml @@ -0,0 +1,56 @@ +# CRUX-O-15 — Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3385, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-15 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3385 + description: > + Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size). Competitor verb: fit(..., infer_limit=0.001, infer_limit_batch_size=10000). Gap measured in aprender at eb262f8eb: No model or ensemble is ever excluded for being slow at predict time; leaderboard timings (O-04) do not exist to compare against. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3385' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + the_selected_ensemble_respects_the_per_row_latency_limit: + formula: | + with infer_limit=L the measured pred_time per row of the final model <= L AND at least one faster-but-worse model was preferred over a slower-but-better one (asserted on a fixture built to force the trade) + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the selected ensemble respects the per-row latency limit" + +falsification_tests: +- id: FALSIFY-CRUX-O-15-001 + rule: "the selected ensemble respects the per-row latency limit" + prediction: "with infer_limit=L the measured pred_time per row of the final model <= L AND at least one faster-but-worse model was preferred over a slower-but-better one (asserted on a fixture built to force the trade)" + test: >- + LIVE-PENDING - the selected ensemble respects the per-row latency limit. No test surface exists today because the capability is unimplemented: Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size) (aprender#3385, CRUX-O-15). PROMOTE by authoring a test named infer_limit_prunes_slow_models in module `automl/budget/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the selected ensemble respects the per-row latency limit' is violated — the autogluon parity claim for CRUX-O-15 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "the selected ensemble respects the per-row latency limit" + +kani_harnesses: +- id: KH-CRUX-O-15-001 + obligation: the_selected_ensemble_respects_the_per_row_latency_limit + property: the_selected_ensemble_respects_the_per_row_latency_limit_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-16-v1.yaml b/contracts/crux-O-16-v1.yaml new file mode 100644 index 0000000000..56c001d51a --- /dev/null +++ b/contracts/crux-O-16-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-16 — Fit diagnostics: fit_summary, model_failures and learning curves +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3386, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-16 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3386 + description: > + Fit diagnostics: fit_summary, model_failures and learning curves. Competitor verb: predictor.fit_summary(); predictor.model_failures(); fit(..., learning_curves=True). Gap measured in aprender at eb262f8eb: ProgressCallback in automl/tuner.rs streams trial results; there is no post-fit summary object listing models trained, models failed with their error, per-model hyperparameters and per-iteration validation curves. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3386' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + a_model_that_raises_during_fit_is_recorded__not_swallowed_an: + formula: | + a portfolio containing a deliberately failing model finishes; model_failures() has exactly one row naming the model and the error string; the leaderboard omits it + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a model that raises during fit is recorded, not swallowed and not fatal" + learning_curves_have_one_point_per_boosting_epoch_iteration: + formula: | + for a GBM with n_estimators=50 and learning_curves=true the curve has 50 validation points, monotone non-increasing after early-stopping's best iteration is asserted absent + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "learning curves have one point per boosting/epoch iteration" + +falsification_tests: +- id: FALSIFY-CRUX-O-16-001 + rule: "a model that raises during fit is recorded, not swallowed and not fatal" + prediction: "a portfolio containing a deliberately failing model finishes; model_failures() has exactly one row naming the model and the error string; the leaderboard omits it" + test: >- + LIVE-PENDING - a model that raises during fit is recorded, not swallowed and not fatal. No test surface exists today because the capability is unimplemented: Fit diagnostics: fit_summary, model_failures and learning curves (aprender#3386, CRUX-O-16). PROMOTE by authoring a test named failing_model_is_recorded_not_fatal in module `automl/summary/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a model that raises during fit is recorded, not swallowed and not fatal' is violated — the autogluon parity claim for CRUX-O-16 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-16-002 + rule: "learning curves have one point per boosting/epoch iteration" + prediction: "for a GBM with n_estimators=50 and learning_curves=true the curve has 50 validation points, monotone non-increasing after early-stopping's best iteration is asserted absent" + test: >- + LIVE-PENDING - learning curves have one point per boosting/epoch iteration. No test surface exists today because the capability is unimplemented: Fit diagnostics: fit_summary, model_failures and learning curves (aprender#3386, CRUX-O-16). PROMOTE by authoring a test named learning_curve_length_matches_iterations in module `automl/summary/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'learning curves have one point per boosting/epoch iteration' is violated — the autogluon parity claim for CRUX-O-16 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "a model that raises during fit is recorded, not swallowed and not fatal" +- type: invariant + property: "learning curves have one point per boosting/epoch iteration" + +kani_harnesses: +- id: KH-CRUX-O-16-001 + obligation: a_model_that_raises_during_fit_is_recorded__not_swallowed_an + property: a_model_that_raises_during_fit_is_recorded__not_swallowed_an_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-16-002 + obligation: learning_curves_have_one_point_per_boosting_epoch_iteration + property: learning_curves_have_one_point_per_boosting_epoch_iteration_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-17-v1.yaml b/contracts/crux-O-17-v1.yaml new file mode 100644 index 0000000000..2c46e58e04 --- /dev/null +++ b/contracts/crux-O-17-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-17 — Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3387, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-17 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3387 + description: > + Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class). Competitor verb: hyperparameters={"TABPFNV2": {}, "TABICL": {}, "MITRA": {}} # in the extreme_quality preset. Gap measured in aprender at eb262f8eb: No tabular in-context learner. nn/ and the inference crates run causal LMs; nothing consumes (X_train, y_train, X_test) as one context and predicts without gradient steps. AutoGluon 1.4-1.6 added TabPFNv2/2.5/2.6/3, TabICL/v2, Mitra, TabDPT, Nori and made them the extreme preset. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3387' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + zero_gradient_prediction_matches_a_pinned_reference_within_t: + formula: | + loading a pinned small checkpoint via apr pull and predicting a 100-row fixture matches the committed reference probabilities within 1e-4 + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "zero-gradient prediction matches a pinned reference within tolerance" + the_context_limit_is_enforced__not_silently_truncated: + formula: | + a context of rows > the model's documented max is an Err naming the limit; a mutation that truncates instead turns this RED + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the context limit is enforced, not silently truncated" + +falsification_tests: +- id: FALSIFY-CRUX-O-17-001 + rule: "zero-gradient prediction matches a pinned reference within tolerance" + prediction: "loading a pinned small checkpoint via apr pull and predicting a 100-row fixture matches the committed reference probabilities within 1e-4" + test: >- + LIVE-PENDING - zero-gradient prediction matches a pinned reference within tolerance. No test surface exists today because the capability is unimplemented: Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) (aprender#3387, CRUX-O-17). PROMOTE by authoring a test named icl_matches_pinned_reference in module `models/tabular_fm/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'zero-gradient prediction matches a pinned reference within tolerance' is violated — the autogluon parity claim for CRUX-O-17 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-17-002 + rule: "the context limit is enforced, not silently truncated" + prediction: "a context of rows > the model's documented max is an Err naming the limit; a mutation that truncates instead turns this RED" + test: >- + LIVE-PENDING - the context limit is enforced, not silently truncated. No test surface exists today because the capability is unimplemented: Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) (aprender#3387, CRUX-O-17). PROMOTE by authoring a test named context_limit_is_an_error in module `models/tabular_fm/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the context limit is enforced, not silently truncated' is violated — the autogluon parity claim for CRUX-O-17 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "zero-gradient prediction matches a pinned reference within tolerance" +- type: invariant + property: "the context limit is enforced, not silently truncated" + +kani_harnesses: +- id: KH-CRUX-O-17-001 + obligation: zero_gradient_prediction_matches_a_pinned_reference_within_t + property: zero_gradient_prediction_matches_a_pinned_reference_within_t_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-17-002 + obligation: the_context_limit_is_enforced__not_silently_truncated + property: the_context_limit_is_enforced__not_silently_truncated_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-18-v1.yaml b/contracts/crux-O-18-v1.yaml new file mode 100644 index 0000000000..6ac97f5bb8 --- /dev/null +++ b/contracts/crux-O-18-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-18 — Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3388, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-18 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3388 + description: > + Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit. Competitor verb: fit(..., memory_limit="auto") # ag.max_memory_usage_ratio; 1.6 calibrated CPU/GPU estimates. Gap measured in aprender at eb262f8eb: No estimator reports an expected peak memory before fitting and nothing checks a limit. AutoGluon 1.6 spent 4 PRs on this (#5757, #5768, #5791, GPU budgeting for parallel folds). + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3388' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + a_model_whose_estimate_exceeds_the_limit_is_skipped_with_rea: + formula: | + with memory_limit=64MiB on a fixture where the estimate for the largest model is > 64MiB, that model is absent from the leaderboard and present in the skip list with its estimate + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a model whose estimate exceeds the limit is skipped with reason=memory" + the_estimate_is_not_a_constant: + formula: | + the estimate for a 10x larger fixture is asserted larger than for the base fixture; a mutation returning a constant turns this RED + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "the estimate is not a constant" + +falsification_tests: +- id: FALSIFY-CRUX-O-18-001 + rule: "a model whose estimate exceeds the limit is skipped with reason=memory" + prediction: "with memory_limit=64MiB on a fixture where the estimate for the largest model is > 64MiB, that model is absent from the leaderboard and present in the skip list with its estimate" + test: >- + LIVE-PENDING - a model whose estimate exceeds the limit is skipped with reason=memory. No test surface exists today because the capability is unimplemented: Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit (aprender#3388, CRUX-O-18). PROMOTE by authoring a test named memory_limit_skips_and_records in module `automl/budget/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a model whose estimate exceeds the limit is skipped with reason=memory' is violated — the autogluon parity claim for CRUX-O-18 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-18-002 + rule: "the estimate is not a constant" + prediction: "the estimate for a 10x larger fixture is asserted larger than for the base fixture; a mutation returning a constant turns this RED" + test: >- + LIVE-PENDING - the estimate is not a constant. No test surface exists today because the capability is unimplemented: Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit (aprender#3388, CRUX-O-18). PROMOTE by authoring a test named estimate_scales_with_data in module `automl/budget/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'the estimate is not a constant' is violated — the autogluon parity claim for CRUX-O-18 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "a model whose estimate exceeds the limit is skipped with reason=memory" +- type: invariant + property: "the estimate is not a constant" + +kani_harnesses: +- id: KH-CRUX-O-18-001 + obligation: a_model_whose_estimate_exceeds_the_limit_is_skipped_with_rea + property: a_model_whose_estimate_exceeds_the_limit_is_skipped_with_rea_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-18-002 + obligation: the_estimate_is_not_a_constant + property: the_estimate_is_not_a_constant_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-19-v1.yaml b/contracts/crux-O-19-v1.yaml new file mode 100644 index 0000000000..4e9ff29c33 --- /dev/null +++ b/contracts/crux-O-19-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-19 — Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3389, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-19 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3389 + description: > + Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq. Competitor verb: TimeSeriesPredictor(prediction_length=48, freq="h").fit(TimeSeriesDataFrame). Gap measured in aprender at eb262f8eb: crates/aprender-core/src/time_series/mod.rs is one struct, ARIMA, on one f32 series (fit/forecast/order). There is no panel container keyed by item and timestamp, no frequency, no horizon-first API and no per-item forecast. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3389' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + a_panel_of_n_items_forecasts_n_x_prediction_length_rows: + formula: | + on a fixture of 5 items with 200 hourly points each and prediction_length=24 the output has exactly 120 rows, each (item_id, timestamp) unique, timestamps continuing each item's last stamp at freq=h + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a panel of N items forecasts N x prediction_length rows" + irregular_timestamps_are_rejected_or_regularised__never_sile: + formula: | + an item with a missing hour is an Err naming the item and gap unless fill=forward is passed, in which case the filled row is flagged + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "irregular timestamps are rejected or regularised, never silently misaligned" + +falsification_tests: +- id: FALSIFY-CRUX-O-19-001 + rule: "a panel of N items forecasts N x prediction_length rows" + prediction: "on a fixture of 5 items with 200 hourly points each and prediction_length=24 the output has exactly 120 rows, each (item_id, timestamp) unique, timestamps continuing each item's last stamp at freq=h" + test: >- + LIVE-PENDING - a panel of N items forecasts N x prediction_length rows. No test surface exists today because the capability is unimplemented: Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq (aprender#3389, CRUX-O-19). PROMOTE by authoring a test named panel_forecast_shape in module `forecast/predictor/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a panel of N items forecasts N x prediction_length rows' is violated — the autogluon parity claim for CRUX-O-19 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-19-002 + rule: "irregular timestamps are rejected or regularised, never silently misaligned" + prediction: "an item with a missing hour is an Err naming the item and gap unless fill=forward is passed, in which case the filled row is flagged" + test: >- + LIVE-PENDING - irregular timestamps are rejected or regularised, never silently misaligned. No test surface exists today because the capability is unimplemented: Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq (aprender#3389, CRUX-O-19). PROMOTE by authoring a test named irregular_index_is_error in module `forecast/predictor/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'irregular timestamps are rejected or regularised, never silently misaligned' is violated — the autogluon parity claim for CRUX-O-19 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "a panel of N items forecasts N x prediction_length rows" +- type: invariant + property: "irregular timestamps are rejected or regularised, never silently misaligned" + +kani_harnesses: +- id: KH-CRUX-O-19-001 + obligation: a_panel_of_n_items_forecasts_n_x_prediction_length_rows + property: a_panel_of_n_items_forecasts_n_x_prediction_length_rows_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-19-002 + obligation: irregular_timestamps_are_rejected_or_regularised__never_sile + property: irregular_timestamps_are_rejected_or_regularised__never_sile_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-20-v1.yaml b/contracts/crux-O-20-v1.yaml new file mode 100644 index 0000000000..694349d2dd --- /dev/null +++ b/contracts/crux-O-20-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-20 — Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3390, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-20 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 5 # 1..5, critical priority in pmat work + intake_status: missing + github_issue: 3390 + description: > + Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE). Competitor verb: TimeSeriesPredictor(eval_metric="WQL", quantile_levels=[0.1,0.5,0.9]). Gap measured in aprender at eb262f8eb: ARIMA.forecast returns a point path; metrics/probabilistic.rs holds classification-probability metrics, and metrics/regression.rs has no seasonal-scaled (MASE/RMSSE) or quantile (WQL/MQL) losses. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3390' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + quantile_forecasts_are_monotone_in_the_quantile_level: + formula: | + for every (item, step) q0.1 <= q0.5 <= q0.9, asserted over the whole fixture; a mutation that shuffles the quantile columns turns this RED + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "quantile forecasts are monotone in the quantile level" + each_metric_matches_the_autogluon_reference_value_on_a_pinne: + formula: | + WQL, MASE, SMAPE, RMSSE, WAPE and MQL computed on a committed (y_true, y_pred, quantiles) fixture equal the reference values produced by autogluon.timeseries.metrics within 1e-9 + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "each metric matches the AutoGluon reference value on a pinned fixture" + +falsification_tests: +- id: FALSIFY-CRUX-O-20-001 + rule: "quantile forecasts are monotone in the quantile level" + prediction: "for every (item, step) q0.1 <= q0.5 <= q0.9, asserted over the whole fixture; a mutation that shuffles the quantile columns turns this RED" + test: >- + LIVE-PENDING - quantile forecasts are monotone in the quantile level. No test surface exists today because the capability is unimplemented: Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) (aprender#3390, CRUX-O-20). PROMOTE by authoring a test named quantiles_monotone in module `forecast/metrics/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'quantile forecasts are monotone in the quantile level' is violated — the autogluon parity claim for CRUX-O-20 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-20-002 + rule: "each metric matches the AutoGluon reference value on a pinned fixture" + prediction: "WQL, MASE, SMAPE, RMSSE, WAPE and MQL computed on a committed (y_true, y_pred, quantiles) fixture equal the reference values produced by autogluon.timeseries.metrics within 1e-9" + test: >- + LIVE-PENDING - each metric matches the AutoGluon reference value on a pinned fixture. No test surface exists today because the capability is unimplemented: Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) (aprender#3390, CRUX-O-20). PROMOTE by authoring a test named metrics_match_autogluon_reference in module `forecast/metrics/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'each metric matches the AutoGluon reference value on a pinned fixture' is violated — the autogluon parity claim for CRUX-O-20 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "quantile forecasts are monotone in the quantile level" +- type: invariant + property: "each metric matches the AutoGluon reference value on a pinned fixture" + +kani_harnesses: +- id: KH-CRUX-O-20-001 + obligation: quantile_forecasts_are_monotone_in_the_quantile_level + property: quantile_forecasts_are_monotone_in_the_quantile_level_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-20-002 + obligation: each_metric_matches_the_autogluon_reference_value_on_a_pinne + property: each_metric_matches_the_autogluon_reference_value_on_a_pinne_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-21-v1.yaml b/contracts/crux-O-21-v1.yaml new file mode 100644 index 0000000000..63608abc91 --- /dev/null +++ b/contracts/crux-O-21-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-21 — Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3391, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-21 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3391 + description: > + Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions. Competitor verb: fit(..., num_val_windows="auto", refit_every_n_windows="auto"); predictor.backtest_predictions(). Gap measured in aprender at eb262f8eb: No time-aware validation. KFold shuffles rows; nothing cuts the last k horizons of each item as expanding-window validation sets, and there is no API that returns the validation forecasts for inspection. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3391' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + validation_windows_never_contain_a_timestamp_later_than_the_: + formula: | + for num_val_windows=3 and val_step_size=h, each window's max training timestamp < min validation timestamp per item, asserted for every item and window + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "validation windows never contain a timestamp later than the training cut for that window" + backtest_predictions_rows_align_with_backtest_targets: + formula: | + the two frames have identical (window, item_id, timestamp) index sets and no NaN targets + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "backtest_predictions rows align with backtest_targets" + +falsification_tests: +- id: FALSIFY-CRUX-O-21-001 + rule: "validation windows never contain a timestamp later than the training cut for that window" + prediction: "for num_val_windows=3 and val_step_size=h, each window's max training timestamp < min validation timestamp per item, asserted for every item and window" + test: >- + LIVE-PENDING - validation windows never contain a timestamp later than the training cut for that window. No test surface exists today because the capability is unimplemented: Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions (aprender#3391, CRUX-O-21). PROMOTE by authoring a test named windows_never_leak_future in module `forecast/backtest/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'validation windows never contain a timestamp later than the training cut for that window' is violated — the autogluon parity claim for CRUX-O-21 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-21-002 + rule: "backtest_predictions rows align with backtest_targets" + prediction: "the two frames have identical (window, item_id, timestamp) index sets and no NaN targets" + test: >- + LIVE-PENDING - backtest_predictions rows align with backtest_targets. No test surface exists today because the capability is unimplemented: Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions (aprender#3391, CRUX-O-21). PROMOTE by authoring a test named backtest_frames_align in module `forecast/backtest/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'backtest_predictions rows align with backtest_targets' is violated — the autogluon parity claim for CRUX-O-21 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "validation windows never contain a timestamp later than the training cut for that window" +- type: invariant + property: "backtest_predictions rows align with backtest_targets" + +kani_harnesses: +- id: KH-CRUX-O-21-001 + obligation: validation_windows_never_contain_a_timestamp_later_than_the_ + property: validation_windows_never_contain_a_timestamp_later_than_the__holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-21-002 + obligation: backtest_predictions_rows_align_with_backtest_targets + property: backtest_predictions_rows_align_with_backtest_targets_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-22-v1.yaml b/contracts/crux-O-22-v1.yaml new file mode 100644 index 0000000000..92d02732ec --- /dev/null +++ b/contracts/crux-O-22-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-22 — Known covariates, past covariates and static features in forecasting +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3392, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-22 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 3 # 1..5, medium priority in pmat work + intake_status: missing + github_issue: 3392 + description: > + Known covariates, past covariates and static features in forecasting. Competitor verb: TimeSeriesPredictor(known_covariates_names=["holiday"]); train_data.static_features = df. Gap measured in aprender at eb262f8eb: ARIMA is univariate. No container carries per-item static features or time-varying covariates, and no model consumes them. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3392' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + a_known_covariate_that_fully_determines_the_target_is_used: + formula: | + on a fixture where y = 10*holiday + noise, a model given known_covariates has MASE < 0.5 x the same model without them + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "a known covariate that fully determines the target is used" + future_covariates_missing_for_the_horizon_is_an_error: + formula: | + predict() without known_covariates for all prediction_length steps is an Err naming the first missing timestamp + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "future covariates missing for the horizon is an error" + +falsification_tests: +- id: FALSIFY-CRUX-O-22-001 + rule: "a known covariate that fully determines the target is used" + prediction: "on a fixture where y = 10*holiday + noise, a model given known_covariates has MASE < 0.5 x the same model without them" + test: >- + LIVE-PENDING - a known covariate that fully determines the target is used. No test surface exists today because the capability is unimplemented: Known covariates, past covariates and static features in forecasting (aprender#3392, CRUX-O-22). PROMOTE by authoring a test named known_covariate_reduces_error in module `forecast/covariates/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'a known covariate that fully determines the target is used' is violated — the autogluon parity claim for CRUX-O-22 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-22-002 + rule: "future covariates missing for the horizon is an error" + prediction: "predict() without known_covariates for all prediction_length steps is an Err naming the first missing timestamp" + test: >- + LIVE-PENDING - future covariates missing for the horizon is an error. No test surface exists today because the capability is unimplemented: Known covariates, past covariates and static features in forecasting (aprender#3392, CRUX-O-22). PROMOTE by authoring a test named missing_future_covariates_is_error in module `forecast/covariates/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'future covariates missing for the horizon is an error' is violated — the autogluon parity claim for CRUX-O-22 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "a known covariate that fully determines the target is used" +- type: invariant + property: "future covariates missing for the horizon is an error" + +kani_harnesses: +- id: KH-CRUX-O-22-001 + obligation: a_known_covariate_that_fully_determines_the_target_is_used + property: a_known_covariate_that_fully_determines_the_target_is_used_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-22-002 + obligation: future_covariates_missing_for_the_horizon_is_an_error + property: future_covariates_missing_for_the_horizon_is_an_error_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-23-v1.yaml b/contracts/crux-O-23-v1.yaml new file mode 100644 index 0000000000..7d76dcf42f --- /dev/null +++ b/contracts/crux-O-23-v1.yaml @@ -0,0 +1,94 @@ +# CRUX-O-23 — Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3393, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-23 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: partial + github_issue: 3393 + description: > + Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston. Competitor verb: hyperparameters={"SeasonalNaive": {}, "AutoETS": {}, "Theta": {}, "AutoARIMA": {}, "Croston": {}}. Gap measured in aprender at eb262f8eb: ARIMA exists (fixed order; no auto-order search). SeasonalNaive, ETS, Theta and the intermittent-demand family (Croston, ADIDA, IMAPA) are absent. AutoGluon runs these as its 'local' tier (timeseries/models/local/) and they anchor every leaderboard. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3393' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + seasonalnaive_is_exactly_the_lag_m_copy: + formula: | + forecast[t] == y[t - m] for the whole horizon on a fixture with m=24, byte-equal + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "SeasonalNaive is exactly the lag-m copy" + autoarima_selects_the_planted_order: + formula: | + on a fixture generated from ARIMA(2,1,1) the selected (p,d,q) equals (2,1,1) in >= 9 of 10 seeds + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "AutoARIMA selects the planted order" + ets_and_theta_match_the_statsforecast_reference: + formula: | + point forecasts on the AirPassengers fixture equal the statsforecast reference within 1e-6 + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "ETS and Theta match the statsforecast reference" + +falsification_tests: +- id: FALSIFY-CRUX-O-23-001 + rule: "SeasonalNaive is exactly the lag-m copy" + prediction: "forecast[t] == y[t - m] for the whole horizon on a fixture with m=24, byte-equal" + test: >- + LIVE-PENDING - SeasonalNaive is exactly the lag-m copy. No test surface exists today because the capability is partial: Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston (aprender#3393, CRUX-O-23). PROMOTE by authoring a test named seasonal_naive_is_lag_copy in module `forecast/local/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'SeasonalNaive is exactly the lag-m copy' is violated — the autogluon parity claim for CRUX-O-23 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-23-002 + rule: "AutoARIMA selects the planted order" + prediction: "on a fixture generated from ARIMA(2,1,1) the selected (p,d,q) equals (2,1,1) in >= 9 of 10 seeds" + test: >- + LIVE-PENDING - AutoARIMA selects the planted order. No test surface exists today because the capability is partial: Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston (aprender#3393, CRUX-O-23). PROMOTE by authoring a test named autoarima_recovers_planted_order in module `forecast/local/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'AutoARIMA selects the planted order' is violated — the autogluon parity claim for CRUX-O-23 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-23-003 + rule: "ETS and Theta match the statsforecast reference" + prediction: "point forecasts on the AirPassengers fixture equal the statsforecast reference within 1e-6" + test: >- + LIVE-PENDING - ETS and Theta match the statsforecast reference. No test surface exists today because the capability is partial: Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston (aprender#3393, CRUX-O-23). PROMOTE by authoring a test named ets_theta_match_statsforecast in module `forecast/local/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'ETS and Theta match the statsforecast reference' is violated — the autogluon parity claim for CRUX-O-23 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "SeasonalNaive is exactly the lag-m copy" +- type: invariant + property: "AutoARIMA selects the planted order" +- type: invariant + property: "ETS and Theta match the statsforecast reference" + +kani_harnesses: +- id: KH-CRUX-O-23-001 + obligation: seasonalnaive_is_exactly_the_lag_m_copy + property: seasonalnaive_is_exactly_the_lag_m_copy_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-23-002 + obligation: autoarima_selects_the_planted_order + property: autoarima_selects_the_planted_order_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-23-003 + obligation: ets_and_theta_match_the_statsforecast_reference + property: ets_and_theta_match_the_statsforecast_reference_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-O-24-v1.yaml b/contracts/crux-O-24-v1.yaml new file mode 100644 index 0000000000..7790b58f59 --- /dev/null +++ b/contracts/crux-O-24-v1.yaml @@ -0,0 +1,75 @@ +# CRUX-O-24 — Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning +# CRUX category O: AutoML Parity (AutoGluon 1.6.3). +# Filed from the 2026-09-16 competitive sweep of ../autogluon. Tracks GitHub #3394, epic #3370. +# registry: false — this contract carries real falsification gates and claims no +# exemption (operator ruling 2026-08-21, "no contract exemptions"). + +metadata: + id: CRUX-O-24 + version: "1.0.0" + created: "2026-09-16" + updated: "2026-09-16" + author: PAIML Engineering + registry: false + status: draft + parent_contracts: + - crux-competitive-research-ux-v1 + category: "O — AutoML Parity" + competitor: autogluon + demand_score: 4 # 1..5, high priority in pmat work + intake_status: missing + github_issue: 3394 + description: > + Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning. Competitor verb: hyperparameters={"Chronos2": {"fine_tune": True}} # 1.5 spotlight; 1.6 adds Toto-2. Gap measured in aprender at eb262f8eb: apr pull fetches LLM checkpoints (Category A) and the inference crates run decoder transformers, but no forecasting head exists: nothing tokenises a numeric series into the model's input, samples a horizon, and maps it back to quantiles. + references: + - 'master: contracts/crux-competitive-research-ux-v1.yaml — §5 + §12' + - 'epic: https://github.com/paiml/aprender/issues/3370' + - 'story: https://github.com/paiml/aprender/issues/3394' + - 'competitor: https://github.com/autogluon/autogluon (1.6.3)' + - 'evidence: evidence/crux/autogluon/api-surface.md' + +equations: + zero_shot_output_matches_a_pinned_reference: + formula: | + a pinned small checkpoint pulled by `apr pull` forecasts the committed fixture within 1e-3 of the reference quantiles + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "zero-shot output matches a pinned reference" + fine_tuning_changes_the_weights_and_improves_the_in_domain_m: + formula: | + after fine_tune on the fixture the WQL improves by >= 5% relative AND at least one weight tensor differs from the pulled checkpoint + domain: "the fixture and inputs named in the prediction" + codomain: "PASS iff the prediction holds; the mutation named in the prediction (where one is named) turns it RED" + invariants: + - "fine-tuning changes the weights and improves the in-domain metric" + +falsification_tests: +- id: FALSIFY-CRUX-O-24-001 + rule: "zero-shot output matches a pinned reference" + prediction: "a pinned small checkpoint pulled by `apr pull` forecasts the committed fixture within 1e-3 of the reference quantiles" + test: >- + LIVE-PENDING - zero-shot output matches a pinned reference. No test surface exists today because the capability is unimplemented: Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning (aprender#3394, CRUX-O-24). PROMOTE by authoring a test named zero_shot_matches_reference in module `forecast/pretrained/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'zero-shot output matches a pinned reference' is violated — the autogluon parity claim for CRUX-O-24 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" +- id: FALSIFY-CRUX-O-24-002 + rule: "fine-tuning changes the weights and improves the in-domain metric" + prediction: "after fine_tune on the fixture the WQL improves by >= 5% relative AND at least one weight tensor differs from the pulled checkpoint" + test: >- + LIVE-PENDING - fine-tuning changes the weights and improves the in-domain metric. No test surface exists today because the capability is unimplemented: Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning (aprender#3394, CRUX-O-24). PROMOTE by authoring a test named finetune_improves_and_changes_weights in module `forecast/pretrained/tests` of crate `aprender-core`; this gate then binds to it and the contract moves from draft to active. Until then the obligation is RECORDED and unfalsifiable-by-absence, NOT satisfied. + if_fails: "rule 'fine-tuning changes the weights and improves the in-domain metric' is violated — the autogluon parity claim for CRUX-O-24 is FALSE and the contract MUST be re-verified against the competitor's canonical implementation" + +proof_obligations: +- type: invariant + property: "zero-shot output matches a pinned reference" +- type: invariant + property: "fine-tuning changes the weights and improves the in-domain metric" + +kani_harnesses: +- id: KH-CRUX-O-24-001 + obligation: zero_shot_output_matches_a_pinned_reference + property: zero_shot_output_matches_a_pinned_reference_holds_on_bounded_input + bound: 4 +- id: KH-CRUX-O-24-002 + obligation: fine_tuning_changes_the_weights_and_improves_the_in_domain_m + property: fine_tuning_changes_the_weights_and_improves_the_in_domain_m_holds_on_bounded_input + bound: 4 diff --git a/contracts/crux-competitive-research-ux-v1.yaml b/contracts/crux-competitive-research-ux-v1.yaml index dd058981e9..6151302d8b 100644 --- a/contracts/crux-competitive-research-ux-v1.yaml +++ b/contracts/crux-competitive-research-ux-v1.yaml @@ -39,6 +39,7 @@ metadata: - "https://github.com/huggingface/transformers" - "https://github.com/vllm-project/vllm" - "https://github.com/mlfoundations/open_clip" + - "https://github.com/autogluon/autogluon" # ───────────────────────────────────────────────────────────── # Demand scoring rubric (see subspec §4) @@ -100,6 +101,11 @@ evidence_sources: capability_matrix: "evidence/crux/openclaw/capability-matrix.yaml" gaps: "evidence/crux/openclaw/gaps.md" interpretation_note: "Resolved 2026-04-18: OpenCLAW is openclaw.ai (local-first personal AI assistant / agent orchestration). NOT OpenCLIP. Category J rewritten accordingly." + autogluon: + readme_verbs: "evidence/crux/autogluon/readme-verbs.txt" + canonical_flow: "evidence/crux/autogluon/hello.sh" + api_surface: "evidence/crux/autogluon/api-surface.md" + interpretation_note: "AutoGluon 1.6.3 (../autogluon @ 77946149), surveyed 2026-09-16. Category O = AutoML parity (tabular + forecasting). MultiModalPredictor CUT on epic #3370." openclip: readme_verbs: "evidence/crux/openclip/readme-verbs.txt" top_issues: "evidence/crux/openclip/top-issues.json" @@ -460,15 +466,46 @@ stories: - { id: CRUX-N-16, title: "Barnes-Hut t-SNE (O(n log n))", competitor: linfa, demand_score: 3, status: partial, contract: crux-N-16-v1.yaml } - { id: CRUX-N-17, title: "Vision ops: NMS, GPU connected components", competitor: burn, demand_score: 2, status: partial, contract: crux-N-17-v1.yaml } + # Category O — AutoML Parity (24) — AutoGluon 1.6.3 (aprender#3370, 2026-09-16). + # Competitor source: ../autogluon at 77946149. Three products (TabularPredictor, + # TimeSeriesPredictor, MultiModalPredictor); MultiModal is CUT on the epic. + # aprender's automl/ is a single-estimator tuner and time_series/ is one ARIMA; + # the gap is the predictor-level UX (fit(label) -> leaderboard -> deploy), not + # the algorithms. Not a BEAT pillar. Registry edit: CRUX_COMPETITORS. + - { id: CRUX-O-01, title: "One-call tabular AutoML: fit(label) -> predict on a CSV", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-01-v1.yaml } + - { id: CRUX-O-02, title: "Problem-type inference: binary / multiclass / regression / quantile fr", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-02-v1.yaml } + - { id: CRUX-O-03, title: "Quality presets (medium / good / high / best / extreme) that name a mo", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-03-v1.yaml } + - { id: CRUX-O-04, title: "Leaderboard: per-model validation/test score, fit time, predict time a", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-04-v1.yaml } + - { id: CRUX-O-05, title: "Automatic feature-type inference and the AutoML feature pipeline (nume", competitor: autogluon, demand_score: 5, status: partial, contract: crux-O-05-v1.yaml } + - { id: CRUX-O-06, title: "K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oo", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-06-v1.yaml } + - { id: CRUX-O-07, title: "Multi-layer stack ensembling with a leakage guard (num_stack_levels, a", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-07-v1.yaml } + - { id: CRUX-O-08, title: "Greedy weighted-ensemble selection over fitted models (Caruana ensembl", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-08-v1.yaml } + - { id: CRUX-O-09, title: "Time-budgeted portfolio fit: time_limit split across models, each mode", competitor: autogluon, demand_score: 5, status: partial, contract: crux-O-09-v1.yaml } + - { id: CRUX-O-10, title: "Predictor-level permutation feature importance with p-values and confi", competitor: autogluon, demand_score: 4, status: partial, contract: crux-O-10-v1.yaml } + - { id: CRUX-O-11, title: "Decision-threshold calibration for binary metrics (calibrate_decision_", competitor: autogluon, demand_score: 4, status: partial, contract: crux-O-11-v1.yaml } + - { id: CRUX-O-12, title: "refit_full: retrain the selected models on train+validation after mode", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-12-v1.yaml } + - { id: CRUX-O-13, title: "Model distillation: compress the ensemble into one fast student", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-13-v1.yaml } + - { id: CRUX-O-14, title: "Deployment artifact: clone_for_deployment / keep_only_best / save_spac", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-14-v1.yaml } + - { id: CRUX-O-15, title: "Inference-latency constraint during model selection (infer_limit, infe", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-15-v1.yaml } + - { id: CRUX-O-16, title: "Fit diagnostics: fit_summary, model_failures and learning curves", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-16-v1.yaml } + - { id: CRUX-O-17, title: "Tabular foundation model: in-context prediction with a pretrained tran", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-17-v1.yaml } + - { id: CRUX-O-18, title: "Memory-aware fit: per-model memory estimate and a memory_limit that sk", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-18-v1.yaml } + - { id: CRUX-O-19, title: "Multi-series forecasting predictor: (item_id, timestamp) panel data, p", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-19-v1.yaml } + - { id: CRUX-O-20, title: "Probabilistic forecasts: quantile_levels and the forecasting metric fa", competitor: autogluon, demand_score: 5, status: missing, contract: crux-O-20-v1.yaml } + - { id: CRUX-O-21, title: "Rolling-window backtesting: num_val_windows, refit_every_n_windows, ba", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-21-v1.yaml } + - { id: CRUX-O-22, title: "Known covariates, past covariates and static features in forecasting", competitor: autogluon, demand_score: 3, status: missing, contract: crux-O-22-v1.yaml } + - { id: CRUX-O-23, title: "Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Cro", competitor: autogluon, demand_score: 4, status: partial, contract: crux-O-23-v1.yaml } + - { id: CRUX-O-24, title: "Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by a", competitor: autogluon, demand_score: 4, status: missing, contract: crux-O-24-v1.yaml } + # ───────────────────────────────────────────────────────────── # Coverage — intake v2.0.0 (verified by awk over §5 of subspec) # ───────────────────────────────────────────────────────────── coverage_intake: supported: 43 - partial: 72 - missing: 152 + partial: 77 + missing: 171 unclear: 0 - total: 267 + total: 291 notes: | ID gaps at C-14, F-10, H-04, I-05, K-06 are intentional — those five stories were dropped pre-v1.0.0 as duplicative (see inline @@ -477,6 +514,9 @@ coverage_intake: 5 partial. It is the first category whose competitors are Rust-native frameworks rather than Python/C++ incumbents, which is why it required a CRUX_COMPETITORS registry edit rather than reuse of an existing source. + Category O (24 rows, aprender#3370) was added 2026-09-16: 19 missing, + 5 partial. Competitor autogluon (1.6.3) — AutoML parity: predictor-level + tabular AutoML and multi-series forecasting. Same registry edit. # ───────────────────────────────────────────────────────────── # Falsification conditions diff --git a/contracts/gguf-kquant-element-size-v1.yaml b/contracts/gguf-kquant-element-size-v1.yaml index 871e3df664..bb76140a9c 100644 --- a/contracts/gguf-kquant-element-size-v1.yaml +++ b/contracts/gguf-kquant-element-size-v1.yaml @@ -1,5 +1,5 @@ metadata: - version: 1.0.0 + version: 1.0.1 created: '2026-06-19' author: PAIML Engineering description: | @@ -25,11 +25,11 @@ metadata: - 'ggml-common.h: sizeof(block_q6_K) = QK_K/2 + QK_K/4 + QK_K/16 + sizeof(ggml_half) = 210' - 'ggml-common.h: sizeof(block_q8_K) = sizeof(float) + QK_K + QK_K/16*sizeof(int16_t) = 292' - 'ggml-common.h: QK_K = 256' - - 'crates/aprender-core/src/format/safetensors.rs — ggml_dtype_element_size SIZES table (idx 10/11/14/15)' + - 'crates/aprender-core/src/format/safetensors.rs — ggml_dtype_element_size, which since #3601 computes type_size / blck_size from trueno_quant::TRAITS (the hand-typed SIZES table is gone)' - 'crates/aprender-core/src/format/tensors.rs — ggml_dtype_name GGML type enum order' name: gguf-kquant-element-size -version: 1.0.0 +version: 1.0.1 scope: aprender-core GGUF K-quant byte-size reporting (safetensors.rs ggml_dtype_element_size) description: | @@ -96,7 +96,7 @@ falsification_tests: 0.8203125 (Q6_K), 1.140625 (Q8_K). Before the fix it returned 0.3125, 0.4375, 0.8125, 1.0625 respectively. test: cargo test -p aprender-core --lib test_kquant_element_size_exact_block_ratio_pmat869 - if_fails: 'a K-quant SIZES table entry (idx 10/11/14/15) regressed to a wrong byte/elem value.' + if_fails: 'a K-quant bytes/elem (id 10/11/14/15) regressed to a wrong value — check ggml_dtype_element_size still reads trueno_quant::TRAITS.' evidence: | RED (buggy table): panics "Q2_K (code 10): expected 0.328125 (= 84/256), got 0.3125". GREEN (fixed table): test passes; 256-element Q2_K tensor reports 84 bytes. diff --git a/contracts/lint-baseline.json b/contracts/lint-baseline.json index 90d48263e5..5230262959 100644 --- a/contracts/lint-baseline.json +++ b/contracts/lint-baseline.json @@ -3,15 +3,15 @@ "armed_gates": ["validate", "audit", "score", "verify", "enforce", "enforcement-level", "duplicate-stems", "composition", "sigma", "relations", "shapes"], "armed_shapes": ["ont-shapes-v1", "ladder-measured", "ladder-green"], "ont": { - "consumer_present": false, - "contracts_total": 1852, - "entity_types_registered": 9, - "extractors_implemented": 6, - "contracts_anchored": 3, - "contracts_shaped": 3, + "consumer_present": true, + "contracts_total": 1881, + "entity_types_registered": 10, + "extractors_implemented": 7, + "contracts_anchored": 6, + "contracts_shaped": 5, "unanchored_but_bindable": 297, "formal_prose": 1464, "legacy_unresolved_depends_on": 8, - "shapes_unarmed": 3 + "shapes_unarmed": 6 } } diff --git a/contracts/ontology.yaml b/contracts/ontology.yaml index e8ad447c68..e026c513a5 100644 --- a/contracts/ontology.yaml +++ b/contracts/ontology.yaml @@ -140,6 +140,10 @@ entity_types: - {name: apr-model, extractor: apr_model, implemented: true} - {name: gguf, extractor: gguf, implemented: true} - {name: csv, extractor: csv, implemented: false} + # ONT-4c3 (aprender#3577): the logit-parity receipts under evidence/parity/**. Subclasses `json` for now; + # it moves under a shared `Receipt` class once quorum and dispatch receipts join, so the three families + # inherit common shapes. NOT the throughput parity receipts — a different family with a different validator. + - {name: parity-receipt, extractor: parity_receipt, implemented: true} extractors: - {name: pv_contract, reader: ontology/extract/pv_contract.rs, implemented: true} @@ -151,6 +155,7 @@ extractors: - {name: apr_model, reader: ontology/extract/apr_model.rs, implemented: true} - {name: gguf, reader: ontology/extract/gguf.rs, implemented: true} - {name: csv, reader: ontology/extract/csv.rs, implemented: false} + - {name: parity_receipt, reader: ontology/extract/parity_receipt.rs, implemented: true} # What this ontology does NOT express, said out loud so nobody encodes it by accident. Each names the reader that # would have to change first. diff --git a/contracts/parity-receipt-v1.yaml b/contracts/parity-receipt-v1.yaml new file mode 100644 index 0000000000..4ca904c25a --- /dev/null +++ b/contracts/parity-receipt-v1.yaml @@ -0,0 +1,120 @@ +# ────────────────────────────────────────────── +# parity-receipt-v1 — the RETIRED logit-parity layout (ONT-001 §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHAT THIS IS. The layout the seven logit-parity records carried from 2026-09-06 until #3577 migrated them: +# the bare `apr parity --json` document, with the readings and the judgment mixed at one level. +# +# {"model": "./m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ … ]} +# +# It is recorded rather than deleted because a version nobody wrote down is a version somebody re-invents. The +# fork was real and cost real time: an extractor keyed to one layout silently skips the other, and skipping is +# indistinguishable from passing — which is how a record can sit in the tree for months with no comparator and +# no shape able to say so. +# +# STATUS: superseded by parity-receipt-v2, WITH NO INSTANCES IN THE TREE. All seven records were migrated to v2 +# in the same commit as the comparator back-fill (#3577), so this contract declares no shape: a shape whose +# target class nothing instantiates passes vacuously, and a vacuous pass is worth less than no shape at all. +# The enforcement that replaces it is stronger and lives where it can fire — `extract:parity-receipt` REFUSES +# an unmigrated record BY NAME (a file with no v2 `schema` but a top-level `metrics[]` or `parity`), and +# `scripts/parity_receipt_denominator.sh` refuses it independently. A returning v1 record is an error naming the +# file, not a silent skip. +# +# WHAT V2 CHANGED, and why each: +# · `schema` — v1 had none, so nothing could tell a parity record from any other JSON in the directory. +# · raw readings separated from the judgment (`raw` vs `result`) — the #3574 layout. `raw` is the original +# document byte for byte, so the migration is a relabel and the measurement is never restated. +# · `comparator {kind, comparator_sha, reason}` — the field whose absence this row exists to make impossible. +# All seven were self-comparisons (apr-CPU vs apr-CUDA) that implied an oracle they never had. +# · `partially_receipted`, `unmeasured[]` — an empty completeness claim now has to be written deliberately. +# · `host`, `backend`, `apr_version`, `generated_at`, `threshold_source` — recovered from committed evidence +# (each record's `provenance.record` names the RECORD.md it was quoted from), never invented. +# · `cell.model_sha256` — carried ONLY by the one record that recorded it at measurement time. The other six +# say so in `unmeasured` instead of being back-filled from a hash taken today. +# +# KIND: pattern (historical). No shape, by construction — see STATUS. +# ────────────────────────────────────────────── +name: parity-receipt-v1 +version: "1.0.0" +scope: > + The retired logit-parity record layout, recorded so the fork it caused is documented and a returning instance + is recognisable. Out of scope: everything live — the shapes, the extractor and the denominator are + parity-receipt-v2's. +status: superseded + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The pre-#3577 logit-parity layout: no schema, readings and judgment at one level, no comparator. Superseded + by parity-receipt-v2, with no instances in the tree; an unmigrated instance is refused by name by the + extractor and by the committed denominator predicate rather than by a shape that would pass vacuously. + references: + - 'aprender#3577 (the migration and the back-fill), #3576, #3269 (ONT-4c3)' + - 'contracts/parity-receipt-v2.yaml — the live contract' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — refuses an unmigrated record by name' + - 'scripts/parity_receipt_denominator.sh — refuses one independently' + - 'evidence/parity/l0-1/{lambda,gx10}/RECORD.md, evidence/parity/l0-1b/gx10/n5/DETERMINISM.md — the provenance the back-fill quotes' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-sigma-v1] + +equations: + # A `kind: pattern` contract declares no SHAPE — see STATUS, a shape over a class nothing + # instantiates passes vacuously — but it still ASSERTS things, and those are statable. These two + # are the invariants below written as the equations the corpus integrity check reads, not new + # claims invented to satisfy a counter: PRC1-INV-001 and PRC1-INV-002 said exactly this already. + no_instances: + formula: "legacy(f) ⇔ f ∈ evidence/parity/**/*.json ∧ ¬f.schema ∧ (f.metrics[] ∨ f.parity)" + domain: every *.json under evidence/parity/, walked in byte order + codomain: the empty set — all seven records were migrated to v2 in the #3577 commit + invariants: + - "the retired layout is recognised by ABSENCE of `schema` plus PRESENCE of a top-level + reading: either half alone is some other document and is not claimed here" + - a count of zero is asserted, not assumed — the denominator predicate measures it independently + preconditions: + - the seven records named in evidence/parity/*/RECORD.md have been migrated + postconditions: + - a v1 record reappearing under evidence/parity/ makes this equation false and is visible + lean_theorem: "none — L4 not declared" + refused_by_name: + formula: "legacy(f) ⇒ f ∈ errors(extract:parity-receipt) ∧ exit(parity_receipt_denominator.sh) ≠ 0" + domain: a planted v1-layout record + codomain: two independent refusals, each naming the file + invariants: + - "refused BY NAME, never skipped: a skip is indistinguishable from a pass, which is how a + record sat in the tree for months with no comparator and nothing able to say so" + - "two readers, not one: the extractor and the shell predicate share no code, so a defect in + either is caught by the other rather than by nobody" + preconditions: + - both readers are wired into CI — a refusal nothing runs is not a refusal + postconditions: + - removing the planted record makes both pass again, so the pair can return to green + lean_theorem: "none — L4 not declared" + +invariants: + - id: PRC1-INV-001 + property: the retired layout has no instances in the committed tree + formal: '∀ f ∈ evidence/parity/**/*.json: schema(f) = "apr-parity-receipt/v2" ∨ ¬legacy(f)' + prose: false + - id: PRC1-INV-002 + property: a returning instance is refused by name, by two independent readers + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors(extract:parity-receipt) ∧ exit(denominator.sh) ≠ 0' + prose: false + +falsification_tests: + - id: FALSIFY-PRC1-001 + rule: no instance survives + prediction: > + planting a v1-layout record under evidence/parity/ makes both the extractor and the denominator predicate + refuse it by name, and removing it makes both pass again — both directions + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt && bash scripts/parity_receipt_denominator.sh --self-test + if_fails: the retired layout returns and nothing says so diff --git a/contracts/parity-receipt-v2.yaml b/contracts/parity-receipt-v2.yaml new file mode 100644 index 0000000000..364562267b --- /dev/null +++ b/contracts/parity-receipt-v2.yaml @@ -0,0 +1,210 @@ +# ────────────────────────────────────────────── +# parity-receipt-v2 — the logit-parity receipt under contract (ONT-001 §3.7, §5 ONT-4c3; issue #3577, PMAT-3577) +# +# WHY THIS EXISTS. Until this contract, the logit-parity records under `evidence/parity/**` had NO validator of +# any kind. Not a weak one — none. That is why seven of them sat in the tree carrying no comparator for months +# and nothing noticed: there was nothing that could have noticed. The operator ruling that opened this row said +# "receipts are the only artifact family in the tree with no shape"; measured, that turned out to be literally +# true of this family. +# +# TWO FAMILIES SHARE THE WORD "PARITY", AND THIS CONTRACT GOVERNS EXACTLY ONE. +# · LOGIT parity (this contract): `apr parity --json`, apr-CPU vs apr-CUDA, one cosine per position. +# · THROUGHPUT parity (NOT this contract): apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 +# cross-class defect. Validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py +# --parity`, which this row deliberately DOES NOT TOUCH. +# The ruling's item 6 said to fold `check_parity_receipt.sh` into this shape. Measured before acting: its +# fixtures require `instrument`, `protocol_ref` and `lanes`, and a logit record has never carried one of them; +# its callers are the dogfood and perf-claim paths. Folding would have deleted the validator for #2696 — the +# published-apr-takes-the-CPU-path-and-reports-0.099x case — from a family nobody was watching. ONE VALIDATOR +# PER ARTIFACT FAMILY, and the discriminator is the artifact's required keys, never its filename. +# +# WHAT IS ARMED. Nothing here, yet. Arming is per shape and lives in `contracts/lint-baseline.json` +# `armed_shapes[]`, a SHARED file this row is forbidden to touch (decision 7). These three shapes are therefore +# COMPUTED AND REPORTED, exactly as `ladder-green` was at ONT-4c1, and arming them is a named follow-up that +# carries the `touches-shared-contracts` label. Reported is not nothing: the gate prints every violation, and +# the back-fill's RED→GREEN is read off that report. +# +# THE COUNT IS PINNED. A shape over an extractor is only as honest as the extractor's reach, and an extractor +# that matches nothing reports the same "no violations" as one that matches everything. So +# `evidence/parity/EXPECTED_RECEIPTS` holds the expected focus-node count, produced by the INDEPENDENT committed +# predicate `scripts/parity_receipt_denominator.sh` (a different implementation of the same question — an +# extractor checked against a number the extractor produced proves nothing). A mismatch, or a refused record, is +# `Unknown{WrongCorpus}` and exit 2: never `Pass`, never a fabricated `Fail`. +# +# NO THRESHOLD IS TYPED HERE. `thresholdSource` is `resolves:` — the extractor resolves the path and +# materialises `thresholdSourceMissing` when it does not exist. The threshold VALUE is read from +# `evidence/parity/thresholds.yaml` by whoever judges; typing one into a shape is this row's STOP condition. +# +# Σ PARENT: `json` (measured — `contracts/ontology.yaml` entity_types carries `{name: json, extractor: json, +# implemented: true}`). It should move under a shared `Receipt` class once quorum and dispatch receipts join the +# graph, so the three families inherit common shapes; R-19 materialisation makes that free. NOT in this row. +# +# KIND: pattern. Vocabulary, an extractor and shapes over a graph; the proof is the gate's own case table. +# ────────────────────────────────────────────── +name: parity-receipt-v2 +version: "2.0.0" +scope: > + How a logit-parity record under evidence/parity/** becomes a parity:ParityReceipt focus node, and the three + shapes over it: the fields every receipt must carry, and the two comparator shapes that differ by kind. Out of + scope: throughput parity receipts (scripts/check_parity_receipt.sh, a different artifact family), quorum and + dispatch receipts (the next entity type), and the threshold VALUES, which are resolved from thresholds.yaml + and never written here. +status: active + +metadata: + version: "2.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + The logit-parity records are focus nodes; every receipt states its comparator, and a receipt that compares + apr against itself says so and why rather than implying an oracle it never had. The extractor's reach is + pinned by a committed denominator so a silent miss cannot read as a clean corpus. + references: + - 'paiml/infra docs/specifications/paiml-ontology.md §3.6 (the implemented SHACL subset), §3.7, §5 ONT-4c3' + - 'aprender#3577 (this row), #3576, #3574/#3575 (the v2 layout), #3269 (ONT-4c3), #3567 (the pv pin)' + - 'crates/aprender-contracts/src/ontology/extract/parity_receipt.rs — the extractor and its case table' + - 'scripts/parity_receipt_denominator.sh — the independent predicate; evidence/parity/EXPECTED_RECEIPTS' + - 'evidence/parity/thresholds.yaml — where a threshold is resolved from, never typed into a shape' + - 'contracts/parity-receipt-v1.yaml — the retired layout these seven records were migrated from' + +entity: + # The entity type this contract's shapes govern (Σ: contracts/ontology.yaml entity_types). It is NOT + # `json`: an `entity: {type: json}` contract must name a document with `entity.ref`, and these two carry + # shapes over an extractor rather than reading one tool's --json output. + type: parity-receipt + +relations: + depends_on: [ont-shapes-v1, ont-relations-v1, ont-sigma-v1, parity-receipt-v1] + +shapes: + # Every receipt, whatever its comparator. `closed` with an EMPTY ignoredProperties is a statement about + # `parity_receipt.rs::emit`: it writes these properties and no others. An ignoredProperties list is a place + # for drift to hide, and after the migration there is one layout, so it is not needed. + - id: parity-receipt-complete + targetClass: parity:ParityReceipt + closed: true + ignoredProperties: [] + properties: + - {path: parity:file, minCount: 1, maxCount: 1} + - {path: parity:host, minCount: 1, maxCount: 1} + - {path: parity:backend, minCount: 1, maxCount: 1, in: [cpu, cuda, wgpu, metal]} + - {path: parity:aprVersion, minCount: 1, maxCount: 1} + - {path: parity:generatedAt, minCount: 1, maxCount: 1} + # `unmeasured` is minCount 1 ON PURPOSE: an empty list is a completeness claim, and a completeness claim + # must be written deliberately (["none"]) rather than arrived at by leaving the key off. + - {path: parity:unmeasured, minCount: 1} + - {path: parity:partiallyReceipted, minCount: 1, maxCount: 1, datatype: xsd:boolean} + - {path: parity:thresholdSource, minCount: 1, maxCount: 1, resolves: path} + - {path: parity:thresholdSourceMissing, maxCount: 0} + # PATTERN, AND DELIBERATELY NO minCount. Six of the seven back-filled records never recorded a model + # hash. Hashing the file on the host today and attaching it to a receipt about 2026-09-06 would be a + # claim about a different world wearing a witness's clothes. Absent means no measurement, never a match + # (ONT-4c1); `partiallyReceipted: true` plus an `unmeasured` entry carries the honesty instead. + - {path: parity:modelSha256, maxCount: 1, pattern: "^[0-9a-f]{64}$"} + - {path: parity:comparator, minCount: 1, maxCount: 1, nodeKind: IRI, class: parity:Comparator, + node: {properties: [{path: parity:kind, minCount: 1, maxCount: 1, + in: [llama_cpp, transformers, self]}]}} + + # The comparator split. The implemented subset has no `sh:or`, so the two cases are two shapes over two + # subclasses the extractor assigns by `comparator.kind` — which is what "two shapes over one sh:node" means + # here. Neither is `closed`: the base shape above owns closure. + - id: parity-comparator-self + targetClass: parity:SelfComparedReceipt + properties: + # A self-comparison must SAY WHY it has no oracle. This is the field that was missing from all seven. + - {path: parity:comparator, node: {properties: [{path: parity:reason, minCount: 1}, + {path: parity:comparatorSha, maxCount: 0}]}} + - id: parity-comparator-oracle + targetClass: parity:OracleComparedReceipt + properties: + # An oracle arm is a claim about another binary; it names which one, or it is not an oracle arm. + - {path: parity:comparator, node: {properties: [{path: parity:comparatorSha, minCount: 1}]}} + +equations: + focus: + formula: "receipt(f) ⇔ f ∈ evidence/parity/**/*.json ∧ f.schema = 'apr-parity-receipt/v2'" + domain: "every *.json under evidence/parity/, walked in byte order" + codomain: "a parity:SelfComparedReceipt node when comparator.kind = self, else a parity:OracleComparedReceipt" + invariants: + - "a file with no v2 schema but a top-level metrics[] or parity is an UNMIGRATED record: refused by name, never skipped" + - "a file that is neither is skipped and counted — skipping is visible, not silent" + preconditions: + - "evidence/parity/EXPECTED_RECEIPTS holds the count the independent predicate measures" + postconditions: + - "two extractions are byte-identical (R-15)" + lean_theorem: none — L4 not declared + reach: + formula: "|focus nodes| ≠ EXPECTED_RECEIPTS ⇒ Unknown{WrongCorpus} ∧ exit = 2" + domain: "one shapes run over the committed tree" + codomain: "a verdict, or a decline that names both numbers" + invariants: + - "a miss is never Pass and never a fabricated Fail: the gate declines and says which two numbers disagree" + - "an ABSENT denominator is a different fault from a broken one and is not folded in here" + preconditions: + - "the denominator is produced by scripts/parity_receipt_denominator.sh, not by the extractor" + postconditions: + - "adding a receipt without updating the denominator declines" + lean_theorem: none — L4 not declared + +invariants: + - id: PRC-INV-001 + property: a receipt with no comparator fails the complete shape + formal: '|comparator(r)| = 0 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-002 + property: a self-comparison without a stated reason fails + formal: 'kind(r) = self ∧ |reason(r)| = 0 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-003 + property: a self-comparison may not name a comparator sha + formal: 'kind(r) = self ∧ |comparatorSha(r)| ≥ 1 ⇒ Fail(parity-comparator-self, r)' + prose: false + - id: PRC-INV-004 + property: an oracle arm names the binary it measured against + formal: 'kind(r) ≠ self ∧ |comparatorSha(r)| = 0 ⇒ Fail(parity-comparator-oracle, r)' + prose: false + - id: PRC-INV-005 + property: a threshold_source that names no file is rejected, and no threshold value is read from the shape + formal: '¬exists(thresholdSource(r)) ⇒ |thresholdSourceMissing(r)| = 1 ⇒ Fail(parity-receipt-complete, r)' + prose: false + - id: PRC-INV-006 + property: the extractor's reach equals the committed denominator or the gate declines + formal: '|focus| ≠ EXPECTED_RECEIPTS ⇒ verdict = Unknown(WrongCorpus)' + prose: false + - id: PRC-INV-007 + property: an unmigrated legacy record is refused, never skipped + formal: 'legacy(f) ∧ ¬schema(f) ⇒ f ∈ errors ∧ f ∉ skipped' + prose: false + +falsification_tests: + - id: FALSIFY-PRC-001 + rule: the extractor's classification + prediction: > + a v2 record becomes one focus node typed by comparator kind; an unmigrated legacy record is refused BY NAME + and is not counted as skipped; an unrelated document (props-*, thresholds) is skipped and counted + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: a record the graph cannot see reads as a clean corpus + - id: FALSIFY-PRC-002 + rule: the pinned reach + prediction: > + committed 2 / found 1, committed 0 / found 1, and a narrowed walk each report WrongCorpus naming both + numbers; committed 1 / found 1 does not; an ABSENT denominator is not a miss + test: cargo test -p aprender-contracts --lib ontology::extract::parity_receipt + if_fails: an extractor that saw the wrong corpus grades it anyway + - id: FALSIFY-PRC-003 + rule: the independent predicate agrees with the extractor + prediction: > + scripts/parity_receipt_denominator.sh measures the same 7 the extractor matches, refuses a planted legacy + record, disagrees when a receipt is added without a bump, and agrees again when it is bumped — both directions + test: bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh + if_fails: the denominator is a number the extractor produced and proves nothing + - id: FALSIFY-PRC-004 + rule: the shapes discriminate + prediction: > + removing `comparator` from a fixture copy raises exactly one violation naming the focus node and the + property; widening `in:` to accept `oracle` turns the mutation RED; the seven back-filled records raise + zero violations and the same seven raise seven before the back-fill + test: cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts + if_fails: the shape decorates the corpus instead of grading it diff --git a/contracts/refusal-receipt-v1.yaml b/contracts/refusal-receipt-v1.yaml new file mode 100644 index 0000000000..f0d98bc34c --- /dev/null +++ b/contracts/refusal-receipt-v1.yaml @@ -0,0 +1,173 @@ +# ────────────────────────────────────────────── +# refusal-receipt-v1 — a refused verb, under contract (issue #3605, PMAT-3605; blocks #3597) +# +# THE DEFECT THIS CLOSES. `removed_by` had **0 occurrences tree-wide**, no schema and no validator. +# Every refusal #3597 writes would have minted an unenforced convention, and a tree full of +# consistent-looking `removed_by:` lines reads as validated when it is decoration — the +# `pv validate`-accepts-a-key-by-ignoring-it class. +# +# ── PHASE 0: WHY ITS OWN CONTRACT, DECIDED FROM Σ AT HEAD ────────────────────────────────────── +# +# The ruling offered two options — `refusal-receipt-v1`, or a field on `parity-receipt-v1` — and +# warned against defaulting to a new contract because it is easier to write. It is not the easier +# one; it is the one the measurements leave standing. +# +# **Option B is refused on a fact: `parity-receipt-v1` DOES NOT EXIST AT HEAD.** It is in unmerged +# PR #3600, and there it is deliberately the RETIRED logit-parity layout carrying **no shape and no +# instances** — every record was migrated to v2. A live, validated field on a superseded contract +# with zero focus nodes is a field nothing can carry. Independently, by the discriminator this tree +# has now used twice (an artifact family is named by its REQUIRED KEYS, never by its filename): a +# parity receipt requires host, backend, comparator, threshold_source and per-position metrics; a +# refusal requires a verb, a reason, an exit code and `removed_by`. **They share no required key.** +# +# **A third option was considered and refused, and it is the one worth writing down** because it is +# more attractive than B: put `removed_by` on `apr-cli-commands-v1.yaml`, where the verb already is +# the focus node and the universe is already the trustworthy 111 (FALSIFY-CLI-001/002). Refused +# because the registry is the **universe**, and #3597's whole method is to DIFF the registry against +# the V / V2 / refusal buckets. If the buckets live inside the registry, the denominator and the +# numerator are the same artifact and the diff is vacuous by construction — a counter measuring its +# own decoration. The registry says what EXISTS; a refusal says what was TRIED and what happened. +# Keeping them apart is what lets that diff be a real diff. +# +# So: its own contract, joined to the registry by `verb`, so a refusal naming a verb the registry +# does not carry is a dangling reference and red. +# +# ── THE FORCED-BINDING TRAP, AND THE ESCAPE THAT MAKES `minCount 1` SAFE ──────────────────────── +# +# A required field with no escape MANUFACTURES FALSE DATA. Two instances were on the table: +# `pv validate` requires a `kani_harnesses` block, so authors fabricate one; and apex#57, where +# `lean_theorem: Theorems.RowMerged` was copied verbatim into TWELVE contracts, resolved to nothing, +# and the gate stayed green throughout. +# +# The fix copied from apex is a **closed set of declared sentinels**: "there is legitimately nothing +# here" must be SAYABLE and still CHECKABLE. `removed_by` is therefore `minCount 1` — every refusal +# must answer — and the answer may be a release OR one of exactly two sentinels, with everything +# else refused as hard as a dangling value: +# +# v. the release that removes this refusal e.g. v0.70 +# never refused permanently and by design; nothing will remove it +# unscheduled a defect or gap with no release chosen yet +# +# `tbd`, `soon`, `n/a`, `pending`, `0.70` (no `v`), `v0.70.1` (a patch is not a release boundary) +# and a git sha are ALL RED. That is the arm that matters: a validator which only accepts cannot +# tell a real value from an invented one, so the falsifier plants a PLAUSIBLE-but-undeclared +# sentinel and requires red. +# +# WHY A VERSION AND NOT A SHA (`done_when` 2 asks for the choice and the reason). A refusal answers +# a user's question — *"which release do I need?"* — and a sha does not answer it. A sha is precise +# about the tree and silent about the boundary; `v0.70` is the thing a person can wait for, and the +# thing `check_milestone_cut.sh` can count. The sha form is made invalid rather than merely +# discouraged, because a shape that permits two spellings gets both. +# +# KIND: pattern. One JSON document, shaped through the `extract:json` path (ONT-001 §3.7) — no new +# extractor: `entity: {type: json, ref}` + `vocabulary:` is exactly the machinery for "validate this +# document", and a second reader for one more family is the thing this tree keeps filing against. +# ────────────────────────────────────────────── +name: refusal-receipt +version: "1.0.0" +scope: > + What a refused verb must state — the verb, one reason line, the exit code a caller sees, and the + release that removes the refusal or a declared sentinel saying none will. Out of scope: WHICH verbs + are refused (that is #3597's bucket over the 111-verb registry), the V / V2 classification, and + parity receipts, which share none of these required keys. +status: active + +metadata: + version: "1.0.0" + kind: pattern + created: '2026-09-20' + last_modified: '2026-09-20' + author: PAIML Engineering + description: > + A refusal is a measurement about a verb: what was tried, what the caller sees, and when it ends. + removed_by is required so no refusal is silent about its own lifetime, and answerable with a + closed sentinel so a required field never manufactures a fabricated release. + references: + - 'aprender#3605 (this row), #3597 (the refusal bucket this unblocks), #3080' + - 'contracts/apr-cli-commands-v1.yaml — the 111-verb registry this joins to by `verb`' + - 'apex#57 — lean_theorem copied into twelve contracts, resolving to nothing, gate green: the forced-binding precedent' + - 'tests/fixtures/ont/refusal-{ok,undeclared-sentinel,sha,missing} — the case table, both arms' + +entity: + type: json + ref: evidence/verbs/refusals.json + +vocabulary: + prefix: refusal + root_class: refusal:Ledger + nested: + refusals: refusal:Refusal + +shape: + targetClass: refusal:Refusal + closed: true + properties: + # The join into the registry. A refusal naming a verb `apr --help` does not carry is a dangling + # reference; the registry is the universe and this field is the only thing pointing at it. + - {path: refusal:verb, minCount: 1, maxCount: 1, datatype: xsd:string} + # One line, and it must say something: #3597 records that a refusal whose text names the wrong + # defect is a fail-open, so an empty reason is refused before a wrong one can be written. + - {path: refusal:reason, minCount: 1, maxCount: 1, datatype: xsd:string, minLength: 12} + # What the CALLER sees. A refusal with no distinct exit code is indistinguishable from success + # to anything that is not reading prose. + - {path: refusal:exit_code, minCount: 1, maxCount: 1, datatype: xsd:integer} + # THE FIELD. Required — every refusal answers — with the closed sentinel set as the escape, so + # "nothing will remove this" is sayable without inventing a release. Anything outside the three + # declared forms is as red as a dangling value. + - {path: refusal:removed_by, minCount: 1, maxCount: 1, + pattern: "^(v[0-9]+\\.[0-9]+|never|unscheduled)$"} + +equations: + answerable: + formula: "∀ r ∈ Refusal: removed_by(r) ∈ {v.} ∪ {never, unscheduled}" + domain: "every entry of evidence/verbs/refusals.json" + codomain: "conforms, or a violation naming the focus node and the value" + invariants: + - "a required field with a closed escape cannot be satisfied by inventing a release" + - "a plausible-but-undeclared sentinel (tbd, soon, n/a, pending) is refused, not tolerated" + - "a sha is refused: it is precise about the tree and silent about the boundary a user waits for" + preconditions: + - "the verb names an entry of contracts/apr-cli-commands-v1.yaml" + postconditions: + - "no refusal is silent about its own lifetime" + lean_theorem: none — L4 not declared + +invariants: + - id: RFS-INV-001 + property: every refusal answers for its own lifetime + formal: '|removed_by(r)| = 0 ⇒ Fail(refusal-receipt-v1, r)' + prose: false + - id: RFS-INV-002 + property: the escape is closed, so the required field cannot manufacture a release + formal: 'removed_by(r) ∉ {v., never, unscheduled} ⇒ Fail(refusal-receipt-v1, r)' + prose: false + - id: RFS-INV-003 + property: a refusal states an exit code a caller can branch on + formal: '|exit_code(r)| = 0 ⇒ Fail(refusal-receipt-v1, r)' + prose: false + - id: RFS-INV-004 + property: a refusal states a reason, so a wrong one is at least visible + formal: 'len(reason(r)) < 12 ⇒ Fail(refusal-receipt-v1, r)' + prose: false + +falsification_tests: + - id: FALSIFY-RFS-001 + rule: both arms of the sentinel set + prediction: > + a refusal carrying `v0.70`, `never` or `unscheduled` conforms; one carrying the PLAUSIBLE but + undeclared `tbd` is refused naming the focus node and the value; so is a git sha, a bare + `0.70`, and a patch-level `v0.70.1` + test: cargo test -p aprender-contracts-cli --test ont_refusal_receipt + if_fails: a validator that only accepts cannot tell a real value from an invented one + - id: FALSIFY-RFS-002 + rule: the required field, with its escape + prediction: > + a refusal with no `removed_by` at all is refused; adding `never` makes it conform without + naming a release that does not exist + test: cargo test -p aprender-contracts-cli --test ont_refusal_receipt + if_fails: the field is required with no escape and starts manufacturing fabricated releases + - id: FALSIFY-RFS-003 + rule: a refusal is legible to a caller + prediction: an entry with no exit_code, or a one-word reason, is refused + test: cargo test -p aprender-contracts-cli --test ont_refusal_receipt + if_fails: a refusal is indistinguishable from success to anything not reading prose diff --git a/contracts/shapes.ttl b/contracts/shapes.ttl index 06b73b15c7..ebfcdfafc0 100644 --- a/contracts/shapes.ttl +++ b/contracts/shapes.ttl @@ -112,3 +112,139 @@ ] ; . + a sh:NodeShape ; + sh:targetClass ; + sh:closed true ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "cpu" "cuda" "wgpu" "metal" ) ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 1 ; + sh:pattern "^[0-9a-f]{64}$" ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class ; + sh:nodeKind sh:IRI ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "llama_cpp" "transformers" "self" ) ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; + sh:property [ + sh:path ; + sh:maxCount 0 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:property [ + sh:path ; + sh:node ; + ] ; +. + + a sh:NodeShape ; + sh:property [ + sh:path ; + sh:minCount 1 ; + ] ; +. + + a sh:NodeShape ; + sh:targetClass ; + sh:closed true ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype ; + sh:minLength 12 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:pattern "^(v[0-9]+\\.[0-9]+|never|unscheduled)$" ; + ] ; +. + diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index 1b66c7aad7..f92636ca7f 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -153,6 +153,10 @@ realizar = { workspace = true, optional = true } # 2026-04-05 PMAT-507: bumped 0.16.0→0.17 to match aprender root Cargo.toml (was diamond dep) trueno = { workspace = true, optional = true } +# The ggml type table (`GgmlType`: ids, names, block sizes), in-tree as crates/aprender-quant. +# Not optional: `apr hex` and `apr validate-manifest` name GGUF dtypes in every build (#3662). +trueno-quant = { workspace = true } + # Async runtime for server tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "sync"], optional = true } diff --git a/crates/apr-cli/src/commands/hex_options_detect.rs b/crates/apr-cli/src/commands/hex_options_detect.rs index cf7c1b8ae4..4e5b14283b 100644 --- a/crates/apr-cli/src/commands/hex_options_detect.rs +++ b/crates/apr-cli/src/commands/hex_options_detect.rs @@ -140,6 +140,20 @@ assert_eq!(ggml_dtype_name(255), "Unknown"); } + /// #3662: the hand-typed table knew 11 ids, so a K-quant below Q4_K, every IQ + /// type and BF16 printed "Unknown" in `apr hex`. + #[test] + fn test_ggml_dtype_name_names_every_family() { + assert_eq!(ggml_dtype_name(10), "Q2_K"); + assert_eq!(ggml_dtype_name(16), "IQ2_XXS"); + assert_eq!(ggml_dtype_name(23), "IQ4_XS"); + assert_eq!(ggml_dtype_name(30), "BF16"); + // Ids ggml removed (Q4_2, Q4_3) are not live types: same token as never-existed. + assert_eq!(ggml_dtype_name(4), "Unknown"); + assert_eq!(ggml_dtype_name(5), "Unknown"); + assert_eq!(ggml_dtype_name(999), "Unknown"); + } + // ======================================================================== // parse_hex_offset // ======================================================================== diff --git a/crates/apr-cli/src/commands/oracle_compute_param_memory.rs b/crates/apr-cli/src/commands/oracle_compute_param_memory.rs index ec47f463ea..5179d689c4 100644 --- a/crates/apr-cli/src/commands/oracle_compute_param_memory.rs +++ b/crates/apr-cli/src/commands/oracle_compute_param_memory.rs @@ -275,6 +275,7 @@ positional_encoding: PositionalEncoding::Absolute, mlp_type: MlpType::GeluMlp, qk_norm: false, + deltanet: None, }, tensor_template: TensorTemplate { embedding: "wte.weight".to_string(), diff --git a/crates/apr-cli/src/commands/oracle_cross_validation_architecture.rs b/crates/apr-cli/src/commands/oracle_cross_validation_architecture.rs index a558db5feb..0c91135949 100644 --- a/crates/apr-cli/src/commands/oracle_cross_validation_architecture.rs +++ b/crates/apr-cli/src/commands/oracle_cross_validation_architecture.rs @@ -273,6 +273,7 @@ positional_encoding: PositionalEncoding::Rope, mlp_type: MlpType::GeluMlp, qk_norm: false, + deltanet: None, }; let params = compute_param_count(&size, &constraints); assert!(params > 0, "Even minimal model should have params"); diff --git a/crates/apr-cli/src/commands/parity_admission.rs b/crates/apr-cli/src/commands/parity_admission.rs index 1b585e9094..370e30b101 100644 --- a/crates/apr-cli/src/commands/parity_admission.rs +++ b/crates/apr-cli/src/commands/parity_admission.rs @@ -387,7 +387,15 @@ mod sentinel_tests { fn record(rel: &str) -> (usize, f32) { let text = std::fs::read_to_string(evidence(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")); let v: serde_json::Value = serde_json::from_str(&text).expect("valid JSON"); - let rows = v["metrics"].as_array().expect("metrics is an array"); + // A v2 receipt (#3577) embeds the raw `apr parity --json` document under `raw`, so the + // readings live at `raw.metrics`; a fresh run carries them at the top level. One rule, + // stated once: look inside the envelope when there is one. + let doc = if v.get("raw").is_some_and(serde_json::Value::is_object) { + &v["raw"] + } else { + &v + }; + let rows = doc["metrics"].as_array().expect("metrics is an array"); let cos: Vec = rows .iter() .filter_map(|r| r["cosine_similarity"].as_f64()) diff --git a/crates/apr-cli/src/commands/run_07.rs b/crates/apr-cli/src/commands/run_07.rs index b121130ada..607f57955c 100644 --- a/crates/apr-cli/src/commands/run_07.rs +++ b/crates/apr-cli/src/commands/run_07.rs @@ -10,4 +10,5 @@ include!("run_tests_inference_output.rs"); include!("run_tests_chrome_trace.rs"); include!("run_tests_layer_trace.rs"); include!("run_tests_stream_output.rs"); +include!("run_tests_accel_reconcile.rs"); } diff --git a/crates/apr-cli/src/commands/run_entry.rs b/crates/apr-cli/src/commands/run_entry.rs index 81524e3e35..e62666df49 100644 --- a/crates/apr-cli/src/commands/run_entry.rs +++ b/crates/apr-cli/src/commands/run_entry.rs @@ -18,6 +18,13 @@ pub(crate) fn run( task: Option<&str>, output_format: &str, no_gpu: bool, + // #3602: the user EXPLICITLY asked for an accelerator, classified by + // `crate::registry::Request::wanted` rather than re-derived here — two + // spellings of one rule is how they drift apart. For THIS command that + // means `--gpu` or `--backend cuda|wgpu|gpu`; `apr run` has no + // `--gpu-layers` flag (that is `apr serve`'s), so the classifier's fourth + // input is genuinely absent here rather than stubbed. + accel_forced: bool, offline: bool, benchmark: bool, verbose: bool, @@ -128,18 +135,114 @@ pub(crate) fn run( print_roofline_profile(&result, max_tokens); } - print_run_output( + // #3602: reconcile what was ASKED FOR with what RAN, then emit. Extracted because + // inlining it took `run`'s cognitive complexity to 27 against a ceiling of 25 — the + // ratchet is measured against origin/main and there is nothing to edit in a baseline + // to make that pass, which is the point of it. + reconcile_and_emit( &result, source, output_format, max_tokens, benchmark, stream, + accel_forced, )?; Ok(()) } +/// Reconcile the requested accelerator against the one that ran, then emit the run's output. +/// +/// The two are one step because their ORDER is the decision: `after_generation`'s contract says a +/// forced refusal prints no output, and #3602 item 1 wants the rejection visible in `--json`. +/// +/// DELIBERATE DEVIATION, named rather than quiet. The no-output rule protects a reader from +/// mistaking a fallback for success. A structured document carrying `"backend": {"fell_back": true}` +/// beside exit 14 cannot be misread that way, while a human-formatted success blob can. So the +/// MACHINE surfaces still emit and the HUMAN surface stays silent — the protective half of the +/// contract is kept, and a `--json` consumer stops having to infer a refusal from an exit code. +/// +/// # Errors +/// [`crate::error::CliError::BackendUnavailable`] when an accelerator was forced and CPU ran, and +/// whatever [`print_run_output`] returns. +#[allow(clippy::too_many_arguments)] +fn reconcile_and_emit( + result: &super::run::RunResult, + source: &str, + output_format: &str, + max_tokens: usize, + benchmark: bool, + stream: bool, + accel_forced: bool, +) -> Result<()> { + let reconciled = reconcile_accelerator(accel_forced, result); + if reconciled.is_ok() || emits_machine_output(stream, output_format, benchmark) { + print_run_output( + result, + source, + output_format, + max_tokens, + benchmark, + stream, + accel_forced, + )?; + } + reconciled +} + +/// Does [`print_run_output`] emit a MACHINE-readable document for these flags? +/// +/// The refusal path above needs to know this, and the first draft answered it +/// with its own copy — `stream || output_format == "json"` — which omitted +/// `!benchmark`. A quorum lane found the consequence: `--json --benchmark` on a +/// refused run took the branch, matched neither machine arm inside +/// `print_run_output`, and fell through to the HUMAN benchmark blob, printing a +/// success rendering for a run being refused. Two spellings of one condition, +/// drifting apart in the gap between them. +/// +/// One spelling now. `the_machine_output_predicate_matches_print_run_output` +/// pins it to the arms it describes over every flag combination, so a change to +/// either side that does not change the other turns the test red. +pub(crate) fn emits_machine_output(stream: bool, output_format: &str, benchmark: bool) -> bool { + !benchmark && (stream || output_format == "json") +} + +/// Compare the accelerator the user ASKED for against the one that RAN. +/// +/// Delegates the decision to [`crate::registry::after_generation`], which is +/// where it is recorded (R-0b, #3002/#3042) and unit-tested: a FORCED +/// accelerator that fell to CPU is a refusal (exit 14, no output), a DEFAULT +/// selection that fell to CPU returns a corrective line to print. +/// +/// **This wires the FORCED half only, and says so rather than carrying a branch +/// that cannot run.** Under `forced = true`, `after_generation` returns either +/// `Err` (the refusal) or `Ok(None)`; its corrective-line branch requires +/// `forced == false` *and* a non-`cpu` announcement, so it is unreachable from +/// here by construction. +/// +/// That case is deliberately not wired. Nothing in the run path calls +/// `registry::announce`, so there is no recorded announcement for a default +/// selection to be compared against, and manufacturing one would assert a +/// choice this process never made. Wiring `announce` is the larger REG-8 job. +/// +/// An earlier draft of this function returned `Result>` and the +/// caller did `if let Some(note) = …`. A quorum lane caught that the `Some` arm +/// could never execute — **a branch with no reachable caller, which is the exact +/// defect this PR exists to fix, reproduced one layer down while fixing it.** +/// +/// # Errors +/// [`crate::error::CliError::BackendUnavailable`] when an accelerator was +/// forced and the generation ran on CPU. +fn reconcile_accelerator(accel_forced: bool, result: &super::run::RunResult) -> Result<()> { + if !accel_forced { + return Ok(()); + } + let _unreachable_here: Option = + crate::registry::after_generation(true, Some("gpu"), result.used_gpu)?; + Ok(()) +} + /// F-CLIPARITY-01 / PMAT-386: Chrome trace JSON output. /// Integrates layer trace + brick profile into chrome://tracing format. /// @@ -355,17 +458,18 @@ fn print_run_output( max_tokens: usize, benchmark: bool, stream: bool, + accel_forced: bool, ) -> Result<()> { // --stream takes precedence — emit JSONL stream. This implies json-style // structured output regardless of --format. (--stream --json is the same // as --stream alone.) if stream && !benchmark { - return print_stream_output(result, source, max_tokens); + return print_stream_output(result, source, max_tokens, accel_forced); } // GH-240/GH-250: JSON output mode with accurate token counts if output_format == "json" && !benchmark { - let json = build_final_json(result, source, max_tokens); + let json = build_final_json(result, source, max_tokens, accel_forced); println!( "{}", serde_json::to_string_pretty(&json).unwrap_or_default() @@ -397,7 +501,12 @@ fn print_run_output( } /// Build the terminal JSON blob shared by `--json` and `--stream` final events. -fn build_final_json(result: &RunResult, source: &str, max_tokens: usize) -> serde_json::Value { +fn build_final_json( + result: &RunResult, + source: &str, + max_tokens: usize, + accel_forced: bool, +) -> serde_json::Value { let tokens_generated = result.tokens_generated.unwrap_or(0); let tok_per_sec = result.tok_per_sec.unwrap_or_else(|| { if result.duration_secs > 0.0 { @@ -418,6 +527,22 @@ fn build_final_json(result: &RunResult, source: &str, max_tokens: usize) -> serd "inference_time_ms": (result.duration_secs * 1000.0 * 100.0).round() / 100.0, "used_gpu": result.used_gpu.unwrap_or(false), "cached": result.cached, + // #3602: `used_gpu: false` alone collapses two different outcomes — "no + // accelerator was asked for" and "one was asked for, attempted, and + // REFUSED at runtime". A consumer cannot tell a CPU run from a rejected + // GPU run, which is how a 33.6 s fallback was read as a GPU timing. + // + // `requested` is what the USER asked for, `ran` is what executed, and + // `fell_back` is true only when those disagree. The rejection's REASON + // (e.g. `cosine 0.4153` at a named position) is on stderr but not yet + // here: it is produced inside realizar's F2 gate and no channel carries + // it to the CLI. Adding one is the #3606-shaped follow-up named in the + // PR — NOT silently approximated with a guess. + "backend": { + "requested": if accel_forced { "gpu" } else { "default" }, + "ran": if result.used_gpu == Some(true) { "gpu" } else { "cpu" }, + "fell_back": accel_forced && result.used_gpu == Some(false), + }, }) } @@ -436,11 +561,16 @@ fn build_final_json(result: &RunResult, source: &str, max_tokens: usize) -> serd /// only when no tokenizer could be resolved for the model; the token id is /// always present and exact, and the terminal `final` event always carries the /// authoritative full text. -fn print_stream_output(result: &RunResult, source: &str, max_tokens: usize) -> Result<()> { +fn print_stream_output( + result: &RunResult, + source: &str, + max_tokens: usize, + accel_forced: bool, +) -> Result<()> { use std::io::Write; let stdout = std::io::stdout(); let mut out = stdout.lock(); - write_stream_output(&mut out, result, source, max_tokens)?; + write_stream_output(&mut out, result, source, max_tokens, accel_forced)?; out.flush()?; Ok(()) } @@ -452,6 +582,7 @@ pub(crate) fn write_stream_output( result: &RunResult, source: &str, max_tokens: usize, + accel_forced: bool, ) -> std::io::Result<()> { if let Some(tokens) = result.generated_tokens.as_deref() { let texts = result.token_texts.as_deref().unwrap_or(&[]); @@ -466,7 +597,7 @@ pub(crate) fn write_stream_output( } } - let mut final_blob = build_final_json(result, source, max_tokens); + let mut final_blob = build_final_json(result, source, max_tokens, accel_forced); if let Some(obj) = final_blob.as_object_mut() { obj.insert( "event".to_string(), diff --git a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs new file mode 100644 index 0000000000..de7f96ad61 --- /dev/null +++ b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs @@ -0,0 +1,190 @@ +// #3602 — `apr run --gpu` must not report a CPU fallback as success. +// +// **The defect, measured on lambda (RTX 4090 sm_89, `apr 0.68.2 (e6f77c98c)`, +// `qwen2.5-coder-0.5b-instruct-q4_k_m`, 2026-09-21):** +// +// ```text +// $ apr run --prompt Hi --max-tokens 1 --gpu --json +// exit=0 +// stderr: warning: GPU output diverges from CPU at position 1 (cosine 0.4153) — falling back to CPU +// stdout: { ..., "used_gpu": false, "inference_time_ms": 33646.13 } +// ``` +// +// Exit 0. Zero `selected:` lines, zero `parity:` lines. A run that was asked for +// the GPU, was refused the GPU at runtime, took 33.6 s on the CPU, and reported +// success — with `used_gpu: false` as the only signal, which is the same value a +// deliberate CPU run reports. +// +// **The mechanism was already built and had no caller.** `registry::after_generation` +// carries the decision (R-0b, #3002/#3042) — forced ⇒ refuse, default ⇒ corrective +// line — with its own unit tests at `registry.rs`, and a `git grep` found its only +// callers were those tests. `registry::announce` and `registry::parity_line` are in +// the same state. `accel.rs` states the rule the whole layer exists to enforce: +// *"a silent CPU fallback is exactly that override wearing a performance number."* +// +// So this file does NOT test a new decision. It tests that the recorded one is +// reachable from `apr run`, which is what was missing. +// +// **Both directions, because "refuse whenever `used_gpu` is false" would pass the +// first test and break every honest CPU run.** +// +// | case | `accel_forced` | `used_gpu` | required | +// |---|---|---|---| +// | asked for GPU, ran on CPU | true | `Some(false)` | **refusal** — the defect | +// | asked for GPU, ran on GPU | true | `Some(true)` | silent pass | +// | asked for nothing, ran on CPU | false | `Some(false)` | silent pass | +// | asked for nothing, ran on GPU | false | `Some(true)` | silent pass | +// | backend did not report | true | `None` | silent pass — absent is not false | +// +// **Round 1 of the AD-04 quorum turned this PR FAIL on two counts, both correct, +// and both are fixed here.** (1) `reconcile_accelerator(...)?` ran BEFORE +// `print_run_output`, so a rejected `--gpu` run early-returned and `--json` +// emitted nothing at all — the surface the ticket names. (2) the caller's +// `if let Some(note)` arm was UNREACHABLE: `announced` was `Some("gpu")` exactly +// when `forced` was true, and `after_generation`'s corrective-line branch needs +// `forced == false`. A branch with no reachable caller — the very defect this PR +// fixes, reproduced one layer down while fixing it. Lane 1 (gemini-3.1-pro-high) +// found both; lane 3 passed the PR without seeing either. +// +// The last row is deliberate. `used_gpu: None` means the engine did not report, +// which is not evidence that it fell back; refusing on it would turn every +// non-reporting path into a hard error. Absence is Unknown, never Fail. + +/// THE CASE THAT SHIPPED. `--gpu`, ran on CPU ⇒ refusal, not a success blob. +#[test] +fn a_forced_accelerator_that_ran_on_cpu_is_refused() { + let result = RunResult { + text: "Hello".to_string(), + duration_secs: 33.646, + cached: true, + tokens_generated: Some(1), + tok_per_sec: Some(0.0), + used_gpu: Some(false), + generated_tokens: Some(vec![9707]), + token_texts: None, + }; + + let err = reconcile_accelerator(true, &result) + .expect_err("--gpu that ran on CPU must refuse, not return a note"); + + // The message has to name the escape hatch, or the refusal is a dead end for + // someone who genuinely wants the CPU run. + let msg = err.to_string(); + assert!( + msg.contains("--no-gpu"), + "the refusal must tell the user how to run on CPU deliberately: {msg}" + ); +} + +/// The falsifier for the fix: a healthy GPU run must stay silent. A build that +/// "always refuses" passes the test above and fails this one. +#[test] +fn a_forced_accelerator_that_actually_ran_on_gpu_says_nothing() { + let result = gpu_result(Some(true)); + reconcile_accelerator(true, &result).expect( + "nothing needs saying when the GPU was asked for and the GPU ran", + ); +} + +/// The other falsifier: an ordinary CPU run was never a fallback. A build that +/// keys on `used_gpu == Some(false)` alone passes the first test and breaks +/// every `apr run` without `--gpu`. +#[test] +fn an_unforced_cpu_run_is_not_a_fallback() { + let result = gpu_result(Some(false)); + reconcile_accelerator(false, &result) + .expect("no accelerator was requested, so there is nothing to reconcile"); +} + +/// Absent is Unknown, never Fail: a backend that did not report `used_gpu` has +/// not reported a fallback. +#[test] +fn a_backend_that_did_not_report_is_not_treated_as_a_fallback() { + let result = gpu_result(None); + reconcile_accelerator(true, &result) + .expect("used_gpu: None is Unknown, not false — not a CPU fallback"); +} + +/// `used_gpu: false` alone collapses "ran on CPU deliberately" and "was refused +/// the GPU". The JSON must separate them, since that is what a consumer reads. +#[test] +fn the_json_distinguishes_a_deliberate_cpu_run_from_a_rejected_gpu_run() { + let cpu = build_final_json(&gpu_result(Some(false)), "m.gguf", 1, false); + let rejected = build_final_json(&gpu_result(Some(false)), "m.gguf", 1, true); + + // Same `used_gpu` — this is exactly the ambiguity that let a 33.6 s CPU + // fallback be read as a GPU timing. + assert_eq!(cpu["used_gpu"], rejected["used_gpu"]); + + assert_eq!(cpu["backend"]["requested"], "default"); + assert_eq!(cpu["backend"]["fell_back"], false); + + assert_eq!(rejected["backend"]["requested"], "gpu"); + assert_eq!(rejected["backend"]["ran"], "cpu"); + assert_eq!( + rejected["backend"]["fell_back"], true, + "a requested GPU that did not run is the whole finding of #3602" + ); +} + +/// A GPU run that succeeded must not be labelled a fallback — the `fell_back` +/// key has to be able to be false while `requested` is `gpu`, or it is +/// decoration rather than a measurement. +#[test] +fn a_successful_gpu_run_is_not_labelled_a_fallback() { + let json = build_final_json(&gpu_result(Some(true)), "m.gguf", 1, true); + assert_eq!(json["backend"]["requested"], "gpu"); + assert_eq!(json["backend"]["ran"], "gpu"); + assert_eq!(json["backend"]["fell_back"], false); +} + +fn gpu_result(used_gpu: Option) -> RunResult { + RunResult { + text: "Hello".to_string(), + duration_secs: 1.0, + cached: true, + tokens_generated: Some(1), + tok_per_sec: Some(1.0), + used_gpu, + generated_tokens: Some(vec![9707]), + token_texts: None, + } +} + +/// The drift guard for `emits_machine_output`. Round 2 of the quorum found +/// `--json --benchmark` leaking a human success blob on a refused run, because +/// the refusal path carried its own copy of the condition and dropped +/// `!benchmark`. This pins the predicate to the arms it describes over EVERY +/// flag combination, so the two cannot diverge again silently. +#[test] +fn the_machine_output_predicate_matches_print_run_output() { + for &stream in &[false, true] { + for &benchmark in &[false, true] { + for fmt in ["json", "text", "table"] { + // Transcribed from `print_run_output`'s own two early returns. + let stream_arm = stream && !benchmark; + let json_arm = fmt == "json" && !benchmark; + assert_eq!( + emits_machine_output(stream, fmt, benchmark), + stream_arm || json_arm, + "predicate disagrees with print_run_output at \ + stream={stream} fmt={fmt} benchmark={benchmark}" + ); + } + } + } +} + +/// The case that leaked, called out by name so a future reader sees the bug and +/// not just the invariant: `--json --benchmark` is NOT a machine surface. +#[test] +fn json_plus_benchmark_is_not_a_machine_surface() { + assert!( + !emits_machine_output(false, "json", true), + "--json --benchmark prints the HUMAN benchmark blob; treating it as a \ + machine surface leaks a success rendering for a refused run" + ); + assert!(emits_machine_output(false, "json", false)); + assert!(emits_machine_output(true, "text", false)); + assert!(!emits_machine_output(true, "json", true)); +} diff --git a/crates/apr-cli/src/commands/run_tests_stream_output.rs b/crates/apr-cli/src/commands/run_tests_stream_output.rs index ea64c8513b..e6062f9a66 100644 --- a/crates/apr-cli/src/commands/run_tests_stream_output.rs +++ b/crates/apr-cli/src/commands/run_tests_stream_output.rs @@ -19,7 +19,7 @@ fn stream_output_emits_n_plus_one_json_lines() { }; let mut buf: Vec = Vec::new(); - write_stream_output(&mut buf, &result, "model.gguf", 32).expect("write must succeed"); + write_stream_output(&mut buf, &result, "model.gguf", 32, false).expect("write must succeed"); let s = String::from_utf8(buf).expect("utf-8"); let lines: Vec<&str> = s.lines().collect(); assert_eq!( @@ -88,7 +88,7 @@ fn stream_token_events_carry_their_own_decoded_text() { }; let mut buf: Vec = Vec::new(); - write_stream_output(&mut buf, &result, "model.gguf", 8).expect("write must succeed"); + write_stream_output(&mut buf, &result, "model.gguf", 8, false).expect("write must succeed"); let s = String::from_utf8(buf).expect("utf-8"); let lines: Vec<&str> = s.lines().collect(); @@ -128,7 +128,7 @@ fn stream_token_events_degrade_to_empty_text_without_a_tokenizer() { }; let mut buf: Vec = Vec::new(); - write_stream_output(&mut buf, &result, "model.apr", 8).expect("write"); + write_stream_output(&mut buf, &result, "model.apr", 8, false).expect("write"); let s = String::from_utf8(buf).expect("utf-8"); let lines: Vec<&str> = s.lines().collect(); assert_eq!(lines.len(), 3, "2 tokens + final, got: {s}"); @@ -153,7 +153,7 @@ fn stream_output_no_tokens_emits_only_final() { }; let mut buf: Vec = Vec::new(); - write_stream_output(&mut buf, &result, "noprompt.apr", 1).expect("write must succeed"); + write_stream_output(&mut buf, &result, "noprompt.apr", 1, false).expect("write must succeed"); let s = String::from_utf8(buf).expect("utf-8"); let lines: Vec<&str> = s.lines().collect(); assert_eq!(lines.len(), 1, "0 tokens + 1 final = 1 line, got: {s}"); @@ -178,7 +178,7 @@ fn stream_output_none_tokens_emits_only_final() { }; let mut buf: Vec = Vec::new(); - write_stream_output(&mut buf, &result, "x.apr", 1).expect("write"); + write_stream_output(&mut buf, &result, "x.apr", 1, false).expect("write"); let s = String::from_utf8(buf).expect("utf-8"); assert_eq!(s.lines().count(), 1); let v: serde_json::Value = @@ -200,7 +200,7 @@ fn build_final_json_matches_legacy_json_shape() { generated_tokens: Some(vec![1, 2, 3]), token_texts: None, }; - let v = build_final_json(&result, "src.apr", 100); + let v = build_final_json(&result, "src.apr", 100, false); assert_eq!(v["model"], "src.apr"); assert_eq!(v["text"], "abc"); assert_eq!(v["tokens"], serde_json::json!([1, 2, 3])); diff --git a/crates/apr-cli/src/commands/sliding_window_entropy.rs b/crates/apr-cli/src/commands/sliding_window_entropy.rs index 4b08f98b12..dd11331a58 100644 --- a/crates/apr-cli/src/commands/sliding_window_entropy.rs +++ b/crates/apr-cli/src/commands/sliding_window_entropy.rs @@ -291,22 +291,10 @@ fn f16_to_f32(h: u16) -> f32 { f32::from_bits((sign << 31) | (exp32 << 23) | mant32) } -/// GGML dtype name from u32 discriminant +/// GGML dtype name from u32 discriminant, read from the ggml type table rather than a +/// copy of it: the hand-typed copy knew 11 ids and printed "Unknown" for the rest (#3662). fn ggml_dtype_name(dtype: u32) -> &'static str { - match dtype { - 0 => "F32", - 1 => "F16", - 2 => "Q4_0", - 3 => "Q4_1", - 6 => "Q5_0", - 7 => "Q5_1", - 8 => "Q8_0", - 9 => "Q8_1", - 12 => "Q4_K", - 13 => "Q5_K", - 14 => "Q6_K", - _ => "Unknown", - } + trueno_quant::GgmlType::from_id(dtype).map_or("Unknown", trueno_quant::GgmlType::as_str) } /// Print an annotated hex field diff --git a/crates/apr-cli/src/commands/tests.rs b/crates/apr-cli/src/commands/tests.rs index b35741ead8..cc54f89b2b 100644 --- a/crates/apr-cli/src/commands/tests.rs +++ b/crates/apr-cli/src/commands/tests.rs @@ -44,6 +44,7 @@ positional_encoding: PositionalEncoding::Rope, mlp_type: MlpType::SwiGlu, qk_norm: false, + deltanet: None, }, tensor_template: TensorTemplate { embedding: "embed.weight".to_string(), @@ -160,6 +161,7 @@ positional_encoding: aprender::format::model_family::PositionalEncoding::Rope, mlp_type: aprender::format::model_family::MlpType::SwiGlu, qk_norm: false, + deltanet: None, }, tensor_template: aprender::format::model_family::TensorTemplate { embedding: String::new(), @@ -301,6 +303,7 @@ positional_encoding: PositionalEncoding::Rope, mlp_type: MlpType::SwiGlu, qk_norm: false, + deltanet: None, } } diff --git a/crates/apr-cli/src/commands/validate_manifest.rs b/crates/apr-cli/src/commands/validate_manifest.rs index 0ee0d81aa1..068192b2f4 100644 --- a/crates/apr-cli/src/commands/validate_manifest.rs +++ b/crates/apr-cli/src/commands/validate_manifest.rs @@ -789,39 +789,10 @@ fn expected_ggml_tensor_type(quant: &str) -> Option { } } +/// The ggml type table's name for `t`, or "UNKNOWN" for an id that is not a live type +/// (removed upstream, or never assigned). Not a hand-typed copy of the table (#3662). fn ggml_type_name(t: u32) -> &'static str { - match t { - 0 => "F32", - 1 => "F16", - 2 => "Q4_0", - 3 => "Q4_1", - 6 => "Q5_0", - 7 => "Q5_1", - 8 => "Q8_0", - 9 => "Q8_1", - 10 => "Q2_K", - 11 => "Q3_K", - 12 => "Q4_K", - 13 => "Q5_K", - 14 => "Q6_K", - 15 => "Q8_K", - 16 => "IQ2_XXS", - 17 => "IQ2_XS", - 18 => "IQ3_XXS", - 19 => "IQ1_S", - 20 => "IQ4_NL", - 21 => "IQ3_S", - 22 => "IQ2_S", - 23 => "IQ4_XS", - 24 => "I8", - 25 => "I16", - 26 => "I32", - 27 => "I64", - 28 => "F64", - 29 => "IQ1_M", - 30 => "BF16", - _ => "UNKNOWN", - } + trueno_quant::GgmlType::from_id(t).map_or("UNKNOWN", trueno_quant::GgmlType::as_str) } /// Pick the "predominant" quantization type from a tensor-type histogram. @@ -1699,6 +1670,20 @@ mod tests { assert_eq!(ggml_type_name(14), "Q6_K"); } + /// #3662: the names come from the ggml type table, not a copy of it. This table + /// was complete when it was hand-typed; the rows pin that the move kept it so, + /// and kept the "UNKNOWN" token for ids that are not live types. + #[test] + fn pm008_ggml_type_name_is_the_upstream_table() { + assert_eq!(ggml_type_name(10), "Q2_K"); + assert_eq!(ggml_type_name(16), "IQ2_XXS"); + assert_eq!(ggml_type_name(23), "IQ4_XS"); + assert_eq!(ggml_type_name(30), "BF16"); + assert_eq!(ggml_type_name(4), "UNKNOWN"); + assert_eq!(ggml_type_name(5), "UNKNOWN"); + assert_eq!(ggml_type_name(999), "UNKNOWN"); + } + #[test] fn pm008_predominant_prefers_non_float() { let mut h = HashMap::new(); diff --git a/crates/apr-cli/src/commands_enum.rs b/crates/apr-cli/src/commands_enum.rs index d608d78d96..8b3e7de36f 100644 --- a/crates/apr-cli/src/commands_enum.rs +++ b/crates/apr-cli/src/commands_enum.rs @@ -136,6 +136,10 @@ pub enum Commands { /// Force GPU acceleration #[arg(long, conflicts_with = "no_gpu")] gpu: bool, + /// Re-run the GPU/CPU parity guard even if a receipt for this (model, + /// apr version, device) exists, and rewrite the receipt (#3604) + #[arg(long)] + revalidate: bool, /// Offline mode: block all network access (Sovereign AI compliance) #[arg(long)] offline: bool, diff --git a/crates/apr-cli/src/dispatch.rs b/crates/apr-cli/src/dispatch.rs index a28c8e0d91..dcd31a50f1 100644 --- a/crates/apr-cli/src/dispatch.rs +++ b/crates/apr-cli/src/dispatch.rs @@ -99,6 +99,41 @@ fn dispatch_sibling_cli_commands(cli: &Cli) -> Option> { } } +/// #3604: `--revalidate` reaches the F2 hybrid guard through the same env seam the guard already +/// reads `SKIP_PARITY_GATE` from (`realizar::gguf::f2_receipt::revalidate_requested`). Chosen over +/// threading a bool through `run_entry::run`'s 37 positional parameters and the six forward +/// signatures that #3606 is changing at the same time; the flag is still a flag to the user, and +/// the guard prints `--revalidate` as its reason when it fires. (Extracted from +/// `dispatch_runtime_commands` unchanged, to keep it under the complexity ratchet.) +fn request_f2_revalidate(revalidate: bool) { + if revalidate { + // SAFETY-BY-ORDER: set before any inference thread exists; the + // only reader is the guard, on this process. + std::env::set_var("APR_F2_REVALIDATE", "1"); + } +} + +/// #3602: classify an `apr run` request ONCE, with the same tested classifier the registry uses. +/// `after_generation` refuses to report a forced accelerator that fell to CPU as success; before +/// this it had no production caller at all, so `apr run --gpu` on a model the GPU gate rejects +/// printed a result and exited 0. (Extracted from `dispatch_runtime_commands` unchanged.) +fn run_accelerator_forced(gpu: bool, no_gpu: bool, backend: Option<&str>) -> bool { + matches!( + crate::registry::Request { + gpu, + no_gpu, + backend, + // `apr run` exposes no `--gpu-layers` (see `Commands::Run` + // in commands_enum.rs — it carries `gpu` and `no_gpu` and + // nothing else); that flag belongs to `apr serve`. So this + // is ABSENT for this surface, not a placeholder to fill in. + layers_want_accelerator: false, + } + .wanted(), + crate::registry::Wanted::Kind(_) | crate::registry::Wanted::AnyAccelerator + ) +} + /// Dispatch runtime commands: check, run, serve. fn dispatch_runtime_commands(cli: &Cli) -> Option> { Some(match cli.command.as_ref() { @@ -117,6 +152,7 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { format, no_gpu, gpu, + revalidate, offline, benchmark, trace, @@ -138,6 +174,7 @@ fn dispatch_runtime_commands(cli: &Cli) -> Option> { verbose, backend: BackendArg { backend }, } => { + request_f2_revalidate(*revalidate); // GH-614: --backend cpu forces CPU-only inference let backend_forces_cpu = backend.as_deref() == Some("cpu"); if let Some(ref b) = backend { @@ -199,6 +236,8 @@ or drop `--backend`." return Some(Err(e)); } + let accel_forced = run_accelerator_forced(*gpu, *no_gpu, backend.as_deref()); + // GH-326: --gpu overrides --no-gpu when both specified let effective_no_gpu = if *gpu { false @@ -233,6 +272,7 @@ or drop `--backend`." task.as_deref(), effective_format, effective_no_gpu, + accel_forced, *offline, *benchmark, *verbose || cli.verbose, diff --git a/crates/apr-cli/src/dispatch_run.rs b/crates/apr-cli/src/dispatch_run.rs index b267120085..350f23d021 100644 --- a/crates/apr-cli/src/dispatch_run.rs +++ b/crates/apr-cli/src/dispatch_run.rs @@ -12,6 +12,9 @@ fn dispatch_run( task: Option<&str>, format: &str, no_gpu: bool, + // #3602: threaded, not re-derived — `dispatch.rs` classifies it once with + // `registry::Request::wanted()`. + accel_forced: bool, offline: bool, benchmark: bool, verbose: bool, @@ -66,6 +69,7 @@ fn dispatch_run( task, format, no_gpu, + accel_forced, offline, benchmark, verbose, diff --git a/crates/apr-cli/src/lib_parse_rosetta.rs b/crates/apr-cli/src/lib_parse_rosetta.rs index 9263616666..a17aba69c8 100644 --- a/crates/apr-cli/src/lib_parse_rosetta.rs +++ b/crates/apr-cli/src/lib_parse_rosetta.rs @@ -303,6 +303,7 @@ format: "text".to_string(), no_gpu: false, gpu: false, + revalidate: false, offline: false, benchmark: false, trace: false, diff --git a/crates/aprender-contracts-cli/src/commands/lint.rs b/crates/aprender-contracts-cli/src/commands/lint.rs index 4cf16a329c..c63ee905fa 100644 --- a/crates/aprender-contracts-cli/src/commands/lint.rs +++ b/crates/aprender-contracts-cli/src/commands/lint.rs @@ -50,6 +50,11 @@ pub fn run( gate: Option<&str>, ) -> Result<(), Box> { refuse_missing_corpus(contract_dir)?; + // Both preconditions before any dispatch: a single file under + // --strict-test-binding is refused (this PR) whether or not --gate (ONT-6) + // is asked for, because run_single_gate would otherwise report every ref + // missing on exactly the input the refusal exists for. + refuse_single_file_strict_binding(contract_dir, strict_test_binding)?; if let Some(name) = gate { return run_single_gate(contract_dir, name); } @@ -287,6 +292,23 @@ fn decide_shapes_gate( reason: Reason::NoFocus, } .into()), + ShapesOutcome::WrongCorpus { + shapes_n, + expected, + found, + refused, + } => { + eprintln!( + "shapes: extract:parity-receipt matched {found} focus node(s); evidence/parity/EXPECTED_RECEIPTS says {expected} ({shapes_n} shape(s))" + ); + for r in &refused { + eprintln!("shapes: refused {r}"); + } + Err(LintDeclined { + reason: Reason::WrongCorpus, + } + .into()) + } ShapesOutcome::NoReceipts { shapes_n, dir } => { // ONT-4c1: the WHY travels with the decline — the lattice has no ReceiptUnmeasured element (ONT-6's // 15 reasons), so the reason is NoCheckable and this line says what could not be checked. @@ -431,6 +453,56 @@ fn show_trend_history(contract_dir: &Path) { /// Permission denied` AHEAD of the refusal: two stderr lines for one decline, /// three under `--diff`. `has_contract_files` does not parse; the post-report /// guard in `run` stays for a corpus that parses to nothing. +/// `pv lint --strict-test-binding` REFUSES rather than reporting a +/// false negative (#3347). +/// +/// The gate resolves cited test names against a source index rooted at the +/// contract path's PARENT. For the directory form that parent is the repo +/// root and the index finds `crates/`; for a single file it is `contracts/`, +/// which holds no source at all, so every cited ref resolves to nothing and +/// every one is reported missing. +/// +/// Measured on `contracts/pv-artifact-kinds-v1.yaml`, a contract whose eight +/// refs all resolve: +/// +/// ```text +/// pv lint contracts/pv-artifact-kinds-v1.yaml --strict-test-binding +/// total_refs 8, existing 0, missing 8 +/// pv lint contracts/ --strict-test-binding +/// total_refs 548, existing 521, missing 27 <- and none of the 27 is this contract +/// ``` +/// +/// A control contract failing identically to a broken one is the definition +/// of a gate that cannot discriminate, so the single-file form is refused. +/// +/// REFUSED rather than repaired: the scan root is computed inside +/// `provable_contracts::lint`, which this ticket does not own. A refusal is +/// in the caller, is honest, and cannot be mistaken for a clean bill. +/// +/// Exit 1, not the exit-2 `decline:` class: exit 2 belongs to `ZeroContracts` +/// and its message ("0 contracts under ...") would be false here -- there IS +/// a contract, it is the gate that cannot run over it. +fn refuse_single_file_strict_binding( + path: &Path, + strict_test_binding: bool, +) -> Result<(), Box> { + if !strict_test_binding || !path.is_file() { + return Ok(()); + } + let dir = path.parent().unwrap_or(Path::new("contracts")); + Err(format!( + "--strict-test-binding cannot run over a single contract file ({}): \ + the gate resolves cited test names against a source tree rooted at \ + that file's parent directory, which holds contracts and no source, \ + so every reference would be reported missing -- including those that \ + do resolve. Run the directory form instead: \ + `pv lint {} --strict-test-binding`.", + path.display(), + dir.display(), + ) + .into()) +} + fn refuse_missing_corpus(contract_dir: &Path) -> Result<(), Box> { if crate::contract_walk::has_contract_files(contract_dir) { return Ok(()); diff --git a/crates/aprender-contracts-cli/tests/includes/cli_binary1.rs b/crates/aprender-contracts-cli/tests/includes/cli_binary1.rs index 73c95f7c7f..58212fd148 100644 --- a/crates/aprender-contracts-cli/tests/includes/cli_binary1.rs +++ b/crates/aprender-contracts-cli/tests/includes/cli_binary1.rs @@ -23,6 +23,75 @@ assert!(!output.status.success()); } + /// #3347: the single-file form of `--strict-test-binding` reported EVERY + /// cited test as missing, because the gate roots its source index at the + /// contract's parent (`contracts/`, which has no `crates/`). Measured on + /// `pv-artifact-kinds-v1.yaml` — a contract whose 8 refs all resolve in + /// the directory form — it reported 8 refs, 0 existing, 8 missing. A + /// control that fails identically to a broken contract cannot + /// discriminate, so the invocation is refused. + #[test] + fn pv_lint_single_file_refuses_strict_test_binding() { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let output = Command::new(pv_bin()) + .current_dir(scratch.path()) + .arg("lint") + .arg(contract_path("pv-artifact-kinds-v1.yaml")) + .arg("--strict-test-binding") + .output() + .expect("failed to run pv"); + assert!( + !output.status.success(), + "the single-file form must be refused, not reported over" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("--strict-test-binding"), + "the refusal must name the flag it refuses: {stderr}" + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("Result: PASS"), + "a PASS was printed over a gate that could not resolve one ref: {stdout}" + ); + assert!( + !stdout.contains("Dangling test reference"), + "the false negatives were printed anyway: {stdout}" + ); + } + + /// Discrimination for the row above: the refusal is specific to the + /// single-FILE form, so a directory is still linted with the same flag. + #[test] + fn pv_lint_directory_form_accepts_strict_test_binding() { + let scratch = tempfile::tempdir().expect("scratch cwd is creatable"); + let dir = scratch.path().join("contracts"); + std::fs::create_dir_all(&dir).expect("fixture dir is creatable"); + std::fs::copy( + contract_path("pv-artifact-kinds-v1.yaml"), + dir.join("pv-artifact-kinds-v1.yaml"), + ) + .expect("fixture contract is copyable"); + + let output = Command::new(pv_bin()) + .current_dir(scratch.path()) + .arg("lint") + .arg(&dir) + .arg("--strict-test-binding") + .output() + .expect("failed to run pv"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("cannot run over a single contract file"), + "the directory form must not be refused: {stderr}" + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("strict-test-binding") || stdout.contains("Result:"), + "the directory form ran the gate and reported: {stdout}" + ); + } + #[test] fn pv_scaffold_softmax() { let output = Command::new(pv_bin()) diff --git a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs index 25c6931414..4b2f19820f 100644 --- a/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs +++ b/crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs @@ -216,10 +216,23 @@ fn the_tracked_repo_graph_is_fresh() { assert_eq!(r.code, 0, "{}", show(&r)); let v = json_of(&r); assert!(v["triples"].as_u64().unwrap_or(0) > 5000, "{}", show(&r)); + // MEASURED on this branch: `pv extract contracts --check` reports + // shapes_n=9, triples=15851 (the 0.69 batch, #3669, pinned pv 0.68.2 built from + // the tree; was 6 / 15620 before #3600 folded in). The count is deliberately hardcoded rather + // than derived — a shape added without anyone noticing is the thing this + // assertion exists to prevent, so adding one is SUPPOSED to turn it red and + // make you name the new shape here. + // + // It did exactly that for #3605's `refusal-receipt-v1`, and a quorum lane + // caught it rather than I did. Note for whoever merges second: this counter + // is shared across branches, so a sibling PR that also adds a shape + // (#3600's parity-receipt-v2) will need the number raised again at merge — + // that is the ratchet working, not a conflict to route around. It did: the + // 0.69 batch folded #3600 in and the count went 6 -> 9 with its three shapes. assert_eq!( v["shapes_n"], - 5, - "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2)\n{}", + 9, + "ont-shapes-v1 + ladder-measured + ladder-green (ONT-4c1) + bound-symbols-resolve + lean-statements-grounded (ONT-4b2) + refusal-receipt-v1 (#3605) + parity-receipt-complete + parity-comparator-self + parity-comparator-oracle (parity-receipt-v2, #3600)\n{}", show(&r) ); } diff --git a/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs new file mode 100644 index 0000000000..ef4f4cde02 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs @@ -0,0 +1,220 @@ +//! ONT-4c3 (PMAT-3577) — logit-parity receipts as focus nodes, on the CLI. +//! +//! The three answers `pv lint --gate shapes` may give about this family, each on its own fixture, so that a +//! build which collapses any two of them fails here: +//! +//! | fixture | exit | why | +//! |---|---|---| +//! | `parity-green` | 0 | a complete v2 receipt: one focus node, no violation | +//! | `parity-nocomparator` | 1 | the state all seven records were in before #3577 back-filled them | +//! | `parity-unknownkind` | 1 | a comparator kind the shape does not accept — this is what the `sh:in` MUTATION breaks | +//! | `parity-unmigrated` | 2 | a legacy record refused BY NAME: `Unknown{WrongCorpus}`, never Pass | +//! | `parity-denominator-drift` | 2 | 2 receipts, denominator 1 — a receipt added without bumping the count | +//! +//! DISCRIMINATION, in both directions. `parity-green` must PASS, so a build that declines every parity corpus +//! fails this file; `parity-nocomparator` and `parity-unknownkind` must FAIL, so a build whose shapes accept +//! anything fails it too. A table of only-invalid cases is passed by a validator that rejects everything. +//! +//! THE MUTATION. Widening `in: [llama_cpp, transformers, self]` to accept `oracle` makes +//! `parity-unknownkind` pass, and this file goes RED. The fixture copies of the contract are asserted +//! byte-identical to `contracts/parity-receipt-v2.yaml`, so mutating the real contract without the fixtures is +//! caught by the drift test instead — either way, RED. A mutation that no committed case can see is not a +//! control. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn fixture(name: &str) -> PathBuf { + repo_root().join("tests/fixtures/ont").join(name) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +impl Run { + fn all(&self) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + self.code, self.stdout, self.stderr + ) + } +} + +fn shapes_on(name: &str) -> Run { + let contracts = fixture(name).join("contracts"); + let out = Command::new(pv_bin()) + .args([ + "lint", + contracts.to_str().expect("utf-8 path"), + "--gate", + "shapes", + "--format", + "json", + ]) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +fn violations(r: &Run) -> usize { + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + v["extra"]["violations"].as_u64().unwrap_or(0) as usize +} + +fn focus_nodes(r: &Run) -> usize { + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + v["extra"]["focus_nodes_n"].as_u64().unwrap_or(0) as usize +} + +#[test] +fn a_complete_receipt_passes_with_one_focus_node() { + let r = shapes_on("parity-green"); + assert_eq!(r.code, 0, "{}", r.all()); + assert_eq!(violations(&r), 0, "{}", r.all()); + assert_eq!(focus_nodes(&r), 1, "{}", r.all()); + // The shapes are ARMED in the fixture (no lint-baseline.json narrows them), so a violation here would be + // a real Fail rather than a reported one — which is what makes the two FAIL cases below meaningful. + assert!(r.stdout.contains("parity-receipt-complete"), "{}", r.all()); +} + +#[test] +fn a_receipt_with_no_comparator_fails_naming_the_property() { + // The state every one of the seven records was in before #3577: a self-comparison implying an oracle. + let r = shapes_on("parity-nocomparator"); + assert_eq!(r.code, 1, "{}", r.all()); + assert_eq!(violations(&r), 1, "exactly one violation\n{}", r.all()); + assert!(r.stdout.contains("parity/comparator"), "{}", r.all()); + assert!(r.stdout.contains("minCount"), "{}", r.all()); +} + +#[test] +fn a_comparator_kind_the_shape_does_not_accept_fails() { + // THE MUTATION TARGET. Widen `in:` to accept `oracle` and this case goes green — so this assertion is + // what makes that mutation detectable. + let r = shapes_on("parity-unknownkind"); + assert_eq!(r.code, 1, "{}", r.all()); + assert!(violations(&r) >= 1, "{}", r.all()); + assert!( + r.stdout.contains("llama_cpp") || r.stdout.contains("\"in\"") || r.stdout.contains("(in)"), + "the report must name the `in` constraint that rejected the kind\n{}", + r.all() + ); +} + +#[test] +fn an_unmigrated_legacy_record_declines_with_exit_2_and_is_named() { + let r = shapes_on("parity-unmigrated"); + assert_eq!(r.code, 2, "a refusal is a DECLINE, not a Fail\n{}", r.all()); + assert!(r.stderr.contains("UNMIGRATED"), "{}", r.all()); + assert!( + r.stderr.contains("legacy.json"), + "named by file\n{}", + r.all() + ); +} + +#[test] +fn a_receipt_added_without_bumping_the_denominator_declines_naming_both_numbers() { + // The falsifier the row names: the extractor found 2, the committed denominator says 1. + let r = shapes_on("parity-denominator-drift"); + assert_eq!(r.code, 2, "{}", r.all()); + assert!(r.stderr.contains("matched 2"), "{}", r.all()); + assert!(r.stderr.contains("says 1"), "{}", r.all()); +} + +#[test] +fn the_three_answers_are_distinct() { + // A build that collapses decline into fail, or fail into pass, fails here rather than in a release. + let codes: Vec = ["parity-green", "parity-nocomparator", "parity-unmigrated"] + .iter() + .map(|f| shapes_on(f).code) + .collect(); + assert_eq!(codes, vec![0, 1, 2], "pass / fail / decline must differ"); +} + +#[test] +fn the_entity_type_is_counted_in_by_entity_type_not_merely_registered() { + // ONT-001 v4.10's probe asks `by_entity_type["parity-receipt"] == 7`. Registering the entity + // type in Σ is not enough for that: without a key here the probe reads ABSENT, and an absent + // key is not zero — a consumer treating it as one measures nothing and calls it a pass. Same + // shape as #3610, one map over. + let out = Command::new(pv_bin()) + .args(["lint", "contracts", "--gate", "shapes", "--format", "json"]) + .current_dir(repo_root()) + .output() + .expect("failed to spawn pv"); + let v: serde_json::Value = + serde_json::from_str(&String::from_utf8_lossy(&out.stdout)).expect("json report"); + let counted = &v["extra"]["by_entity_type"]["parity-receipt"]; + assert!( + !counted.is_null(), + "by_entity_type carries no `parity-receipt` key — the probe would read ABSENT, not 7" + ); + assert_eq!( + counted.as_u64(), + v["extra"]["by_shape"] + .as_array() + .expect("by_shape") + .iter() + .find_map(|s| s + .as_str()? + .strip_prefix("parity-receipt-complete=")? + .parse() + .ok()), + "the entity count and the shape's focus-node count must be the same number" + ); +} + +#[test] +fn every_fixture_carries_the_real_contract_byte_for_byte() { + // A fixture copy that drifts from `contracts/parity-receipt-v2.yaml` would let the real shape be mutated + // while the case table stayed green — the mutation control's blind spot, closed here. + let real = std::fs::read(repo_root().join("contracts/parity-receipt-v2.yaml")) + .expect("the real contract is in the tree"); + for name in [ + "parity-green", + "parity-nocomparator", + "parity-unknownkind", + "parity-unmigrated", + "parity-denominator-drift", + ] { + let copy = std::fs::read(fixture(name).join("contracts/parity-receipt-v2.yaml")) + .unwrap_or_else(|e| panic!("{name} carries the contract: {e}")); + assert_eq!( + copy, real, + "{name}'s copy of parity-receipt-v2.yaml has drifted from contracts/" + ); + } +} + +#[test] +fn the_committed_tree_agrees_with_its_own_denominator() { + // The real corpus, not a fixture: the independent predicate and the extractor must reach the same count. + let out = Command::new("bash") + .arg("scripts/parity_receipt_denominator.sh") + .current_dir(repo_root()) + .output() + .expect("failed to spawn the denominator predicate"); + assert!( + out.status.success(), + "exit {:?}\n{}{}", + out.status.code(), + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); +} diff --git a/crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs b/crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs new file mode 100644 index 0000000000..b58eb2d007 --- /dev/null +++ b/crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs @@ -0,0 +1,189 @@ +//! #3605 / PMAT-3605 — `removed_by` under contract, on the CLI. +//! +//! **Both arms, and the second one is the point.** A validator that only ever sees valid input is +//! indistinguishable from `exit 0`, so the table plants a **plausible but undeclared** sentinel +//! (`tbd`) and requires RED. Accepting `v0.70` proves nothing on its own. +//! +//! | fixture | `removed_by` | expected | +//! |---|---|---| +//! | `refusal-ok` | `v0.70`, `never`, `unscheduled` | **Pass** — the declared set, all three forms | +//! | `refusal-undeclared-sentinel` | `tbd` | Fail — the arm that matters | +//! | `refusal-sha` | `ddb5a15eb` | Fail — precise about the tree, silent about the boundary | +//! | `refusal-bare-version` | `0.70` | Fail — a shape that permits two spellings gets both | +//! | `refusal-patch-version` | `v0.70.1` | Fail — a patch is not a release boundary | +//! | `refusal-missing` | *(absent)* | Fail — every refusal answers for its own lifetime | +//! | `refusal-no-exit-code` | — | Fail — a refusal a caller cannot branch on | +//! | `refusal-terse-reason` | — | Fail — `reason: "broken"` names no defect | +//! +//! **Why `minCount 1` is safe here and manufactures nothing.** A required field with no escape +//! produces fabricated data — `pv validate` requires `kani_harnesses`, so authors invent one, and +//! apex#57 had `lean_theorem: Theorems.RowMerged` copied verbatim into twelve contracts, resolving +//! to nothing, gate green throughout. The escape is the closed sentinel set: `never` and +//! `unscheduled` let "there is legitimately nothing here" be SAID and still CHECKED. That is why +//! `refusal-ok` carries all three forms and `refusal-undeclared-sentinel` must be red — the escape +//! has to be open enough to be honest and closed enough to be a gate. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn pv_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pv")) +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn fixture(name: &str) -> PathBuf { + repo_root().join("tests/fixtures/ont").join(name) +} + +struct Run { + code: i32, + stdout: String, + stderr: String, +} + +impl Run { + fn all(&self) -> String { + format!( + "exit {}\n--- stdout\n{}\n--- stderr\n{}", + self.code, self.stdout, self.stderr + ) + } +} + +fn shapes_on(name: &str) -> Run { + let contracts = fixture(name).join("contracts"); + let out = Command::new(pv_bin()) + .args([ + "lint", + contracts.to_str().expect("utf-8 path"), + "--gate", + "shapes", + "--format", + "json", + ]) + .output() + .expect("failed to spawn pv"); + Run { + code: out.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + } +} + +/// Every fixture whose `removed_by` is outside the declared set must FAIL, and the report must name +/// the focus node — a red that does not say which entry is a red nobody can act on. +fn assert_refused(name: &str) { + let r = shapes_on(name); + assert_eq!(r.code, 1, "{name} was not refused\n{}", r.all()); + assert!( + r.stdout.contains("refusal-receipt-v1"), + "{name}: the report must name the shape that fired\n{}", + r.all() + ); +} + +#[test] +fn all_three_declared_forms_conform() { + // A release, and BOTH sentinels. If the escape did not work, a refusal with nothing to promise + // would have to invent a release — which is the defect this contract exists to prevent. + let r = shapes_on("refusal-ok"); + assert_eq!(r.code, 0, "{}", r.all()); + let v: serde_json::Value = serde_json::from_str(&r.stdout).expect("json report"); + assert_eq!(v["extra"]["violations"].as_u64(), Some(0), "{}", r.all()); + assert!( + v["extra"]["by_shape"] + .as_array() + .expect("by_shape") + .iter() + .any(|s| s.as_str() == Some("refusal-receipt-v1=3")), + "all three entries must be focus nodes\n{}", + r.all() + ); +} + +#[test] +fn a_plausible_but_undeclared_sentinel_is_refused() { + // THE ARM THAT MATTERS. `tbd` is exactly what someone writes when the field is required and + // they have nothing to say; if it passed, the closed set would be decoration. + assert_refused("refusal-undeclared-sentinel"); +} + +#[test] +fn a_sha_is_refused_because_it_answers_a_different_question() { + assert_refused("refusal-sha"); +} + +#[test] +fn a_shape_that_permits_two_spellings_would_get_both() { + assert_refused("refusal-bare-version"); + assert_refused("refusal-patch-version"); +} + +#[test] +fn a_refusal_that_says_nothing_about_its_lifetime_is_refused() { + assert_refused("refusal-missing"); +} + +#[test] +fn a_refusal_a_caller_cannot_branch_on_or_read_is_refused() { + assert_refused("refusal-no-exit-code"); + assert_refused("refusal-terse-reason"); +} + +#[test] +fn accept_and_refuse_are_distinct_answers() { + // A build that collapses them passes an all-invalid table and an all-valid one alike. + assert_eq!(shapes_on("refusal-ok").code, 0); + assert_eq!(shapes_on("refusal-undeclared-sentinel").code, 1); +} + +#[test] +fn every_fixture_carries_the_real_contract_byte_for_byte() { + // Without this, the shape could be widened in `contracts/` while the case table stayed green — + // the mutation control's blind spot. + let real = std::fs::read(repo_root().join("contracts/refusal-receipt-v1.yaml")) + .expect("the real contract is in the tree"); + for name in [ + "refusal-ok", + "refusal-undeclared-sentinel", + "refusal-sha", + "refusal-bare-version", + "refusal-patch-version", + "refusal-missing", + "refusal-no-exit-code", + "refusal-terse-reason", + ] { + let copy = std::fs::read(fixture(name).join("contracts/refusal-receipt-v1.yaml")) + .unwrap_or_else(|e| panic!("{name} carries the contract: {e}")); + assert_eq!( + copy, real, + "{name}'s copy of refusal-receipt-v1.yaml has drifted from contracts/" + ); + } +} + +#[test] +fn every_refusal_in_the_tree_names_a_verb_the_registry_carries() { + // The join the contract declares: the registry is the universe, and a refusal naming a verb + // `apr --help` does not expose is a dangling reference. Checked here rather than in the shape + // because the implemented subset cannot reach across documents. + let ledger: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(repo_root().join("evidence/verbs/refusals.json")) + .expect("the ledger is in the tree"), + ) + .expect("the ledger is JSON"); + let registry = std::fs::read_to_string(repo_root().join("contracts/apr-cli-commands-v1.yaml")) + .expect("the registry is in the tree"); + let refusals = ledger["refusals"].as_array().expect("refusals[]"); + assert!(!refusals.is_empty(), "an empty ledger passes vacuously"); + for r in refusals { + let verb = r["verb"].as_str().expect("verb is a string"); + assert!( + registry.contains(&format!("- name: {verb}\n")), + "refusal names verb {verb:?}, which contracts/apr-cli-commands-v1.yaml does not carry" + ); + } +} diff --git a/crates/aprender-contracts/src/lint/mod.rs b/crates/aprender-contracts/src/lint/mod.rs index 47f2950b46..db3b347077 100644 --- a/crates/aprender-contracts/src/lint/mod.rs +++ b/crates/aprender-contracts/src/lint/mod.rs @@ -737,6 +737,10 @@ fn shapes_result(contract_dir: &Path, validation_passed: bool) -> (GateResult, V skipped_gate("shapes", &format!("{shapes_n} shape(s), no focus node")), Vec::new(), ), + shapes_gate::ShapesOutcome::WrongCorpus { shapes_n, expected, found, refused } => ( + skipped_gate("shapes", &format!("extract:parity-receipt matched {found} focus node(s) and evidence/parity/EXPECTED_RECEIPTS says {expected} ({shapes_n} shape(s)){} — an extractor that saw the wrong corpus reports the same \"no violations\" as one that saw all of it", if refused.is_empty() { String::new() } else { format!("; refused: {}", refused.join("; ")) })), + Vec::new(), + ), shapes_gate::ShapesOutcome::NoReceipts { shapes_n, dir } => ( skipped_gate("shapes", &format!("{shapes_n} shape(s) resolve receipts and the tree holds none under {dir}/ — R-2: unmeasured is a decline")), Vec::new(), diff --git a/crates/aprender-contracts/src/lint/shapes_gate.rs b/crates/aprender-contracts/src/lint/shapes_gate.rs index e813127d18..feab19f457 100644 --- a/crates/aprender-contracts/src/lint/shapes_gate.rs +++ b/crates/aprender-contracts/src/lint/shapes_gate.rs @@ -59,6 +59,16 @@ pub enum ShapesOutcome { NoFocus { shapes_n: usize }, /// A shape resolves receipts and the tree holds none under `evidence/dogfood/models/`. NoReceipts { shapes_n: usize, dir: String }, + /// PMAT-3577 — `extract:parity-receipt` matched a different number of focus nodes than + /// `evidence/parity/EXPECTED_RECEIPTS` says the tree holds, or it refused a record by name. An + /// extractor that silently sees the wrong corpus reports the same "no violations" as one that sees + /// all of it, so this is `Unknown{WrongCorpus}` — never `Pass`, never a fabricated `Fail`. + WrongCorpus { + shapes_n: usize, + expected: usize, + found: usize, + refused: Vec, + }, /// A positive control did not fire. PositiveControlFailed { shapes_n: usize, @@ -121,6 +131,66 @@ fn armed_shapes_of(contract_dir: &Path) -> Result { }) } +/// PMAT-3577: the parity corpus must be exactly what `EXPECTED_RECEIPTS` pins, and no record may be refused, +/// before anything is graded. Either miss is `WrongCorpus`, never a corpus verdict. (Extracted from +/// `run_shapes_gate` unchanged, to keep it under the complexity ratchet.) +fn parity_refusal( + parity: &extract::parity_receipt::ParityStats, + shapes_n: usize, +) -> Option { + let refused = || parity.errors.iter().map(ToString::to_string).collect(); + if let Some((expected, found)) = parity.wrong_corpus() { + return Some(ShapesOutcome::WrongCorpus { + shapes_n, + expected, + found, + refused: refused(), + }); + } + if !parity.errors.is_empty() { + return Some(ShapesOutcome::WrongCorpus { + shapes_n, + expected: parity.expected.unwrap_or(parity.records), + found: parity.records, + refused: refused(), + }); + } + None +} + +/// A torn JSONL line is the INPUT's fault and is carried as a warning on the entity's root shape — the +/// gate already rules that warnings alone are `Unknown{Warn}`, never a pass and never a silent drop. +fn carry_extract_warnings(report: &mut Report, warnings: &[extract::json::Warning]) { + for w in warnings { + report.results.push(shapes::ValidationResult { + severity: Severity::Warning, + focus: iri("json", &w.contract), + shape: w.contract.clone(), + path: Some(w.path.clone()), + component: "extract:json", + message: w.to_string(), + }); + } +} + +/// ONT-4b2: the vendored W3C cases, every run. A validator that fails the standard's own case for a form +/// it claims has no standing to grade the corpus: `Err(Differential)`. `Ok` carries the run, whose counts +/// the verdict reports. +fn w3c_checked(shapes_n: usize, focus_nodes_n: usize) -> Result { + let w3c_run = w3c::run_all(); + let failed = w3c_run.failed(); + if failed.is_empty() { + return Ok(w3c_run); + } + Err(ShapesOutcome::Differential { + shapes_n, + focus_nodes_n, + passed: w3c_run.passed(), + n: w3c_run.results.len(), + failed, + }) +} + /// Run the gate over `contract_dir`. #[must_use] pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { @@ -146,6 +216,10 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { Ok(x) => x, Err(e) => return ShapesOutcome::ExtractFailed(e), }; + // PMAT-3577: the count is pinned before anything is graded. A miss here is not a corpus verdict. + if let Some(refusal) = parity_refusal(&extraction.parity, shapes.len()) { + return refusal; + } let graph = extraction.graph; let needs_receipts = shapes.iter().any(|s| { s.properties @@ -154,18 +228,7 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { }); let (mut report, plant_violations) = validate_with_plant(&graph, &shapes, &arming); - // A torn JSONL line is the INPUT's fault and is carried as a warning on the entity's root shape — the - // gate already rules that warnings alone are `Unknown{Warn}`, never a pass and never a silent drop. - for w in &extraction.warnings { - report.results.push(shapes::ValidationResult { - severity: Severity::Warning, - focus: iri("json", &w.contract), - shape: w.contract.clone(), - path: Some(w.path.clone()), - component: "extract:json", - message: w.to_string(), - }); - } + carry_extract_warnings(&mut report, &extraction.warnings); let pc_extract = extract_controls(); let unmeasured = needs_receipts && extraction.receipts.is_empty(); if let Some(d) = decline( @@ -177,19 +240,10 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { ) { return d; } - // ONT-4b2: the vendored W3C cases, every run. A validator that fails the standard's own case for a form - // it claims has no standing to grade the corpus. - let w3c_run = w3c::run_all(); - let w3c_failed = w3c_run.failed(); - if !w3c_failed.is_empty() { - return ShapesOutcome::Differential { - shapes_n: shapes.len(), - focus_nodes_n: report.focus_nodes_n, - passed: w3c_run.passed(), - n: w3c_run.results.len(), - failed: w3c_failed, - }; - } + let w3c_run = match w3c_checked(shapes.len(), report.focus_nodes_n) { + Ok(run) => run, + Err(differential) => return differential, + }; let counted = findings_of( &report, @@ -230,6 +284,11 @@ pub fn run_shapes_gate(contract_dir: &Path) -> ShapesOutcome { extraction.gguf.rungs.len() + extraction.gguf.files_read, ), ("apr-model", extraction.apr_model.files_read), + // ONT-4c3: registered in Σ and implemented, so it is counted here like every other entity + // type. Without this key a probe asking `by_entity_type["parity-receipt"]` reads ABSENT — + // and an absent key is not zero, so a consumer that treats it as one measures nothing and + // calls it a pass. The same shape as #3610, one map over. + ("parity-receipt", extraction.parity.records), ("code", extraction.code.symbols), ("lean", extraction.lean.statements), ] diff --git a/crates/aprender-contracts/src/lint/strict_test_binding.rs b/crates/aprender-contracts/src/lint/strict_test_binding.rs index c2c30449a4..c9e9bc5c81 100644 --- a/crates/aprender-contracts/src/lint/strict_test_binding.rs +++ b/crates/aprender-contracts/src/lint/strict_test_binding.rs @@ -959,6 +959,7 @@ fn ignored_test_still_counts() {} test_harness: Some(harness.into()), name: Some(name.into()), if_fails: "investigate".into(), + ..Default::default() }); vec![("fixture".to_string(), c)] } diff --git a/crates/aprender-contracts/src/obligation_matrix.rs b/crates/aprender-contracts/src/obligation_matrix.rs index 60634b8bd8..040e3429c7 100644 --- a/crates/aprender-contracts/src/obligation_matrix.rs +++ b/crates/aprender-contracts/src/obligation_matrix.rs @@ -7,6 +7,44 @@ use serde::{Deserialize, Serialize}; use crate::proof_status::ProofLevel; use crate::schema::Contract; +/// What the L2 column knows about ONE obligation (#3347). +/// +/// Three values, not two, because "no test covers this" and "nothing in this +/// contract says which test covers what" are different facts and only one of +/// them is a finding. The old column had no way to say the second, so it said +/// the first -- as a tick. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum L2Status { + /// A falsification test in this contract CITES this obligation. + Tested, + /// This contract's obligation-to-test links resolve, and none names this + /// obligation -- or the contract ships no falsification test at all. + Untested, + /// Nothing readable links any test to any obligation here, so whether this + /// obligation is tested was not measured. An unread window is Unknown, + /// never a tick and never a failure. + Unknown, +} + +impl L2Status { + /// `true` only for [`Tested`](L2Status::Tested) -- an Unknown is not a pass. + #[must_use] + pub fn is_tested(self) -> bool { + matches!(self, Self::Tested) + } + + /// The table cell for this verdict. + #[must_use] + pub fn mark(self) -> &'static str { + match self { + Self::Tested => "\u{2713}", + Self::Untested => "\u{2717}", + Self::Unknown => "?", + } + } +} + /// Verification status for a single obligation across all proof levels. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ObligationStatus { @@ -14,8 +52,10 @@ pub struct ObligationStatus { pub property: String, /// Obligation type (invariant, bound, equivalence, etc.) pub obligation_type: String, - /// Whether at least one falsification test covers this obligation - pub l2_tested: bool, + /// Whether a falsification test is LINKED to this obligation -- and + /// whether that question could be answered at all. Replaces the old + /// `l2_tested: bool`, which could not distinguish the two (#3347). + pub l2: L2Status, /// Whether at least one Kani harness covers this obligation pub l3_kani: bool, /// Whether the obligation has a Lean proof with status "proved" @@ -33,16 +73,128 @@ pub struct ContractObligationMatrix { pub obligations: Vec, } +/// Which obligations of one contract a falsification test actually cites. +/// +/// `resolved` counts citations that named something findable. Zero of them is +/// the difference between "this contract says nothing is tested" and "this +/// contract says nothing at all", and only the second is Unknown. +struct LinkTally { + /// Per obligation index: some test cites it. + cited: Vec, + /// Citations that resolved to a real obligation / a real test. + resolved: usize, +} + +/// Position of the obligation a test citation names: its `id`, else its exact +/// `property` text. Both spellings occur in `contracts/` (12 and 6 entries). +fn obligation_index(contract: &Contract, token: &str) -> Option { + contract.proof_obligations.iter().position(|ob| { + ob.id.as_deref().is_some_and(|id| id.trim() == token) || ob.property.trim() == token + }) +} + +/// Whether a `discharged_by` token names a falsification test that EXISTS. +/// +/// `falsification_tests[N]` resolves only when `N` is in range -- an +/// out-of-range index is a dangling citation, not a proof. A token naming a +/// kani harness resolves here as false: kani is the L3 column. +fn cites_existing_test(contract: &Contract, token: &str) -> bool { + if let Some(inner) = token + .strip_prefix("falsification_tests[") + .and_then(|rest| rest.strip_suffix(']')) + { + return inner + .trim() + .parse::() + .is_ok_and(|i| i < contract.falsification_tests.len()); + } + contract + .falsification_tests + .iter() + .any(|ft| ft.id.trim() == token) +} + +/// Resolve every obligation-to-test link this contract declares, in both +/// spellings: `falsification_tests[].obligation` (alias `binds_to`) and +/// `proof_obligations[].discharged_by`. +fn tally_links(contract: &Contract) -> LinkTally { + let mut tally = LinkTally { + cited: vec![false; contract.proof_obligations.len()], + resolved: 0, + }; + + for ft in &contract.falsification_tests { + let Some(citation) = ft.obligation.as_ref() else { + continue; + }; + for token in citation.targets() { + if let Some(idx) = obligation_index(contract, token) { + tally.cited[idx] = true; + tally.resolved += 1; + } + } + } + + for (idx, ob) in contract.proof_obligations.iter().enumerate() { + let Some(citation) = ob.discharged_by.as_ref() else { + continue; + }; + for token in citation.targets() { + if cites_existing_test(contract, token) { + tally.cited[idx] = true; + tally.resolved += 1; + } + } + } + + tally +} + +/// The L2 verdict for one obligation. See [`L2Status`]. +fn l2_verdict(contract: &Contract, tally: &LinkTally, idx: usize) -> L2Status { + // No falsification test exists, so no test covers this. That is a reading, + // not an unread window. + if contract.falsification_tests.is_empty() { + return L2Status::Untested; + } + // Tests exist but nothing links any of them to any obligation -- or every + // link this contract declares dangles. Either way the window is unread. + if tally.resolved == 0 { + return L2Status::Unknown; + } + if tally.cited.get(idx).copied().unwrap_or(false) { + L2Status::Tested + } else { + L2Status::Untested + } +} + /// Build per-obligation verification matrices for a list of contracts. /// /// For each contract, determines per-obligation coverage: -/// - **L2**: A falsification test covers this obligation (index-based or rule match) +/// - **L2**: a falsification test is LINKED to this obligation -- see +/// [`l2_verdict`] and [`L2Status`] /// - **L3**: A Kani harness references this obligation (property match) /// - **L4**: The obligation has a `lean` field with `status: proved` +/// +/// # The L2 column used to tick on a count (#3347) +/// +/// It read `idx < falsification_tests.len()`: obligation 3 was "tested" +/// because the contract had at least 4 tests, whoever those tests were about. +/// All 7 obligations of `qwen35-e2e-verification-v1` showed a tick before a +/// single test existed. The fallback -- a substring match between the +/// obligation's `property` and a test's `rule` prose -- is gone too: an +/// inferred overlap of two English sentences is not a claim that the test +/// proves the obligation, and it ticked on words like "count". +/// +/// Measured consequence over `contracts/`, and it is the point rather than a +/// regression: L2 ticks fall from 3,573 of 3,753 obligation rows to the +/// handful the corpus actually binds. The rest report `?`. pub fn obligation_matrix(contracts: &[(String, &Contract)]) -> Vec { contracts .iter() .map(|(stem, contract)| { + let tally = tally_links(contract); let obligations = contract .proof_obligations .iter() @@ -50,14 +202,7 @@ pub fn obligation_matrix(contracts: &[(String, &Contract)]) -> Vec Vec Vec String for ob in &matrix.obligations { let check = "\u{2713}"; let cross = "\u{2717}"; - let l2 = if ob.l2_tested { check } else { cross }; + let l2 = ob.l2.mark(); let l3 = if ob.l3_kani { check } else { cross }; let l4 = if ob.l4_lean { check } else { cross }; let prop_display = truncate(&ob.property, max_prop_width); @@ -159,17 +304,28 @@ pub fn format_obligation_table(matrices: &[ContractObligationMatrix]) -> String out } -/// Check whether two property descriptions share significant words. +/// Truncate `s` to at most `max` BYTES, cutting on a char boundary. /// -/// Splits both strings into words (>= 3 chars, excluding stop words) and +/// #3338: this sliced `&s[..max]` and `pv proof-status contracts/ --table` +/// panicked on the real corpus — `byte index 40 is not a char boundary; it is +/// inside '∈'`. The column width is a byte count (`property.len()`), so the +/// cut lands mid-char for any property holding a multi-byte char near it; +/// eight contracts in `contracts/` do. The budget stays a byte budget (the +/// table is laid out in bytes) and the cut walks back to the nearest boundary. pub fn truncate(s: &str, max: usize) -> &str { if s.len() <= max { - s - } else { - &s[..max] + return s; + } + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; } + &s[..end] } +/// Check whether two property descriptions share significant words. +/// +/// Splits both strings into words (>= 3 chars, excluding stop words) and /// returns true if at least one non-trivial word overlaps. pub fn property_words_match(a: &str, b: &str) -> bool { let stop_words: &[&str] = &[ diff --git a/crates/aprender-contracts/src/ontology/extract/mod.rs b/crates/aprender-contracts/src/ontology/extract/mod.rs index 053e8f0d1e..3f2c54939c 100644 --- a/crates/aprender-contracts/src/ontology/extract/mod.rs +++ b/crates/aprender-contracts/src/ontology/extract/mod.rs @@ -19,6 +19,7 @@ pub mod code; pub mod gguf; pub mod json; pub mod lean; +pub mod parity_receipt; pub mod pv_contract; /// Every extractor's output over `contract_dir`, plus the input-side warnings the extractors chose to carry @@ -41,6 +42,8 @@ pub struct Extraction { pub code: code::CodeStats, /// ONT-4b2: the in-tree Lean theorems and the contracts that cite them. pub lean: lean::LeanStats, + /// ONT-4c3: the logit-parity receipts under `evidence/parity/**`, and the files this extractor refused. + pub parity: parity_receipt::ParityStats, } /// What a walk could not do. Every variant is the DECLARATION's fault (exit 3), never a corpus verdict. @@ -87,6 +90,7 @@ pub fn all(contract_dir: &Path) -> Result { out.resolve = receipts::resolve(&mut out.graph, &out.gguf.rungs, &out.receipts); out.code = code::extract(contract_dir, &mut out.graph); out.lean = lean::extract(contract_dir, &mut out.graph); + out.parity = parity_receipt::extract(root, &mut out.graph); Ok(out) } diff --git a/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs b/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs new file mode 100644 index 0000000000..1a9c0968a0 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/parity_receipt.rs @@ -0,0 +1,303 @@ +//! ONT-001 §3.7, §5 ONT-4c3 — `extract:parity-receipt`: a logit-parity record under +//! `evidence/parity/**` becomes a `parity:ParityReceipt` focus node (PMAT-3577, aprender#3577). +//! +//! **Which family this is.** Two artifact families in this tree share the word "parity". This module reads the +//! **logit** family — `apr parity --json`, CPU vs CUDA, a cosine per position — and nothing else. The +//! **throughput** family (apr vs llama.cpp tok/s, `lanes[]`, `decode_tok_per_sec`, the #2696 cross-class +//! defect) is validated by `scripts/check_parity_receipt.sh` over `scripts/lib/bench_receipt.py --parity`, +//! which this row deliberately does NOT touch: its fixtures require `instrument`, `protocol_ref` and +//! `lanes`, none of which a logit record has ever carried. One validator per artifact family — folding one +//! into the other would delete coverage from whichever family nobody was watching. +//! +//! **Why this exists at all.** Until this row, the logit records had NO validator of any kind. That is why +//! seven of them sat in the tree with no comparator for months and nothing noticed: there was nothing that +//! could have noticed. +//! +//! **Three outcomes per file, and skipping is never one of them silently.** Every `*.json` under +//! `evidence/parity/**` is one of: +//! +//! | class | rule | consequence | +//! |---|---|---| +//! | record | `schema` == [`SCHEMA`] | a focus node | +//! | **unmigrated** | no v2 schema, but a top-level `metrics[]` or `parity` | **refused BY NAME** — never skipped | +//! | other | neither | skipped, counted in [`ParityStats::skipped`] | +//! +//! The middle row is the point. A file that looks like a parity record and carries no schema is the legacy +//! layout this row migrated away; leaving one behind would be a record the graph cannot see, and an extractor +//! that silently skips is indistinguishable from one that passes. +//! +//! **The count is pinned, because an extractor that matches nothing reports the same "no violations" as one +//! that matches everything.** [`EXPECTED_FILE`] holds the expected focus-node count, produced by the committed +//! predicate `scripts/parity_receipt_denominator.sh`. A mismatch is `Unknown{WrongCorpus}`, exit 2 — never +//! `Pass`, never a fabricated `Fail`. + +use std::path::{Path, PathBuf}; + +use crate::ontology::rdf::{iri, Graph, Term, RDF_TYPE}; +use crate::ontology::shapes::RDFS_SUBCLASS_OF; + +/// The one layout this reader accepts. The legacy layout has no `schema` key at all and is refused by name. +pub const SCHEMA: &str = "apr-parity-receipt/v2"; +/// Where the logit-parity records live, relative to the repository root. +pub const EVIDENCE_DIR: &str = "evidence/parity"; +/// The committed denominator: the focus-node count the extractor must reproduce. +pub const EXPECTED_FILE: &str = "evidence/parity/EXPECTED_RECEIPTS"; + +/// The vocabulary root for parity receipts: `https://ont.paiml.dev/v1alpha1/parity/`. +#[must_use] +pub fn parity(name: &str) -> String { + format!("{}parity/{name}", crate::ontology::rdf::ONT_BASE) +} + +/// A record this reader refuses, by name. A refusal is never a corpus verdict — it is this file's fault. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParityError { + pub file: String, + pub what: String, +} + +impl std::fmt::Display for ParityError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.file, self.what) + } +} + +impl std::error::Error for ParityError {} + +/// What the walk found. `records` is what the denominator pins. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ParityStats { + /// Files carrying [`SCHEMA`] — one focus node each. + pub records: usize, + /// Files under the tree that are not parity records at all (`props-*`, `derived_expiries`, …). + pub skipped: usize, + /// Records whose `threshold_source` names a path that is not in the tree. + pub threshold_source_missing: usize, + /// Files refused by name: an unmigrated legacy record, or a record this reader cannot read. + pub errors: Vec, + /// The committed expectation, when [`EXPECTED_FILE`] is present and parses. + pub expected: Option, +} + +impl ParityStats { + /// The extractor matched a different number of records than the committed denominator says. + /// + /// `None` when the denominator is absent — that is a different fault (a missing declaration), reported + /// by the caller, and deliberately not folded in here: "no expectation" and "a broken expectation" are + /// not the same state. + #[must_use] + pub fn wrong_corpus(&self) -> Option<(usize, usize)> { + match self.expected { + Some(n) if n != self.records => Some((n, self.records)), + _ => None, + } + } +} + +/// Read the committed denominator: the first non-comment, non-blank line, parsed as a count. +fn read_expected(root: &Path) -> Option { + let text = std::fs::read_to_string(root.join(EXPECTED_FILE)).ok()?; + text.lines() + .map(str::trim) + .find(|l| !l.is_empty() && !l.starts_with('#')) + .and_then(|l| l.parse().ok()) +} + +/// A document that carries no v2 `schema` but has the legacy layout's fingerprint: a top-level `metrics` +/// array, or a top-level `parity` key. Such a file is an unmigrated record, not an unrelated document. +fn looks_legacy(v: &serde_json::Value) -> bool { + v.get("metrics").is_some_and(serde_json::Value::is_array) || v.get("parity").is_some() +} + +/// Every `*.json` under `/evidence/parity/**`, in byte order, classified and — for records — emitted. +pub fn extract(root: &Path, g: &mut Graph) -> ParityStats { + let mut stats = ParityStats { + expected: read_expected(root), + ..ParityStats::default() + }; + let mut files = Vec::new(); + walk(&root.join(EVIDENCE_DIR), &mut files); + files.sort(); + if !files.is_empty() { + declare_classes(g); + } + for f in files { + let rel = f + .strip_prefix(root) + .unwrap_or(&f) + .to_string_lossy() + .replace('\\', "/"); + let Ok(text) = std::fs::read_to_string(&f) else { + stats.errors.push(ParityError { + file: rel, + what: "unreadable".into(), + }); + continue; + }; + let Ok(v) = serde_json::from_str::(&text) else { + // Not JSON at all under a tree of JSON: say so rather than counting it as "other". + stats.errors.push(ParityError { + file: rel, + what: "not JSON".into(), + }); + continue; + }; + match v.get("schema").and_then(serde_json::Value::as_str) { + Some(SCHEMA) => { + emit(g, root, &rel, &v, &mut stats); + stats.records += 1; + } + other if looks_legacy(&v) => { + stats.errors.push(ParityError { + file: rel, + what: format!( + "an UNMIGRATED logit-parity record: schema {} is not {SCHEMA}. The legacy layout \ + was migrated by #3577; a record the extractor cannot see is a record no shape \ + can refuse.", + other.map_or_else(|| "absent".to_string(), |s| format!("{s:?}")) + ), + }); + } + _ => stats.skipped += 1, + } + } + stats +} + +fn walk(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + walk(&p, out); + } else if p.extension().and_then(|x| x.to_str()) == Some("json") { + out.push(p); + } + } +} + +/// `SelfComparedReceipt` and `OracleComparedReceipt` are subclasses of `ParityReceipt`, so a shape targeting +/// the parent sees every receipt while the two child shapes carry the constraints that differ by comparator +/// kind — the subset has no `sh:or`, and two shapes over one class is how that split is expressed. +fn declare_classes(g: &mut Graph) { + for child in ["SelfComparedReceipt", "OracleComparedReceipt"] { + g.insert( + parity(child), + RDFS_SUBCLASS_OF.to_string(), + Term::iri(parity("ParityReceipt")), + ); + } +} + +fn s(v: &serde_json::Value, k: &str) -> Option { + v.get(k).and_then(|x| match x { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Null => None, + other => Some(other.to_string()), + }) +} + +/// One record → one focus node. Every property the shapes name is written here; nothing else is, so +/// `closed: true` with an empty `ignoredProperties` is a statement about this function. +fn emit(g: &mut Graph, root: &Path, rel: &str, v: &serde_json::Value, stats: &mut ParityStats) { + let node = iri("parity-receipt", rel); + let comparator = v.get("comparator"); + let kind = comparator.and_then(|c| s(c, "kind")).unwrap_or_default(); + let class = if kind == "self" { + "SelfComparedReceipt" + } else { + "OracleComparedReceipt" + }; + g.insert(node.clone(), RDF_TYPE, Term::iri(parity(class))); + g.insert(node.clone(), parity("file"), Term::string(rel)); + for (key, prop) in [ + ("host", "host"), + ("backend", "backend"), + ("apr_version", "aprVersion"), + ("generated_at", "generatedAt"), + ("threshold_source", "thresholdSource"), + ] { + if let Some(val) = s(v, key) { + g.insert(node.clone(), parity(prop), Term::string(val)); + } + } + if let Some(b) = v + .get("partially_receipted") + .and_then(serde_json::Value::as_bool) + { + g.insert(node.clone(), parity("partiallyReceipted"), Term::boolean(b)); + } + if let Some(sha) = v.get("cell").and_then(|c| s(c, "model_sha256")) { + g.insert(node.clone(), parity("modelSha256"), Term::string(sha)); + } + for u in v + .get("unmeasured") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + { + g.insert(node.clone(), parity("unmeasured"), Term::string(u)); + } + // `resolves:` is ours, not SHACL: the extractor does the resolving and materialises the failure as a + // literal a shape can refuse with `maxCount 0`. The THRESHOLD VALUE is never read here — a threshold + // typed into a shape instead of resolved from thresholds.yaml is this row's STOP condition. + if let Some(src) = s(v, "threshold_source") { + if !root.join(&src).exists() { + stats.threshold_source_missing += 1; + g.insert( + node.clone(), + parity("thresholdSourceMissing"), + Term::string(src), + ); + } + } + if let Some(c) = comparator { + let cnode = iri("parity-comparator", rel); + g.insert(cnode.clone(), RDF_TYPE, Term::iri(parity("Comparator"))); + g.insert(cnode.clone(), parity("kind"), Term::string(&kind)); + if let Some(sha) = s(c, "comparator_sha") { + g.insert(cnode.clone(), parity("comparatorSha"), Term::string(sha)); + } + if let Some(r) = s(c, "reason") { + g.insert(cnode.clone(), parity("reason"), Term::string(r)); + } + g.insert(node, parity("comparator"), Term::iri(cnode)); + } +} + +/// The positive control (R-3): a copy of a real record with `comparator` removed must lose its comparator +/// edge, every run. Drawn beside the corpus so "the extractor still reads this layout" is measured rather +/// than assumed. +#[must_use] +pub fn positive_control(sample: &serde_json::Value) -> bool { + let mut g = Graph::new(); + let mut stats = ParityStats::default(); + let root = Path::new("."); + emit(&mut g, root, "__pc_sample__", sample, &mut stats); + let with = !g + .objects( + &iri("parity-receipt", "__pc_sample__"), + &parity("comparator"), + ) + .is_empty(); + let mut stripped = sample.clone(); + if let Some(o) = stripped.as_object_mut() { + o.remove("comparator"); + } + let mut g2 = Graph::new(); + emit(&mut g2, root, "__pc_planted__", &stripped, &mut stats); + let without = g2 + .objects( + &iri("parity-receipt", "__pc_planted__"), + &parity("comparator"), + ) + .is_empty(); + with && without +} + +#[cfg(test)] +#[path = "parity_receipt_tests.rs"] +mod tests; diff --git a/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs b/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs new file mode 100644 index 0000000000..4ea4a66f20 --- /dev/null +++ b/crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs @@ -0,0 +1,230 @@ +//! PMAT-3577 — the extractor's case table. Every case is a state that either happened in this tree or is +//! one edit away from happening, and each says which. + +use super::*; + +fn record(kind: &str, extra: serde_json::Value) -> serde_json::Value { + let mut v = serde_json::json!({ + "schema": SCHEMA, + "cell": {"model": "m", "file": "./m.gguf", "quant": "Q4_K_M"}, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": {"kind": kind, "reason": "no oracle arm exists for this cell"}, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": ["ORACLE ARM: not measured."], + "result": {"positions": 78, "parity": true}, + "raw": {"model": "./m.gguf", "metrics": []} + }); + if let (Some(o), Some(e)) = (v.as_object_mut(), extra.as_object()) { + for (k, val) in e { + o.insert(k.clone(), val.clone()); + } + } + v +} + +fn emit_one(v: &serde_json::Value) -> (Graph, ParityStats) { + let mut g = Graph::new(); + let mut stats = ParityStats::default(); + emit(&mut g, Path::new("."), "r.json", v, &mut stats); + (g, stats) +} + +fn objects(g: &Graph, prop: &str) -> Vec { + g.objects(&iri("parity-receipt", "r.json"), &parity(prop)) + .iter() + .filter_map(|t| t.as_literal().map(|l| l.0.to_string())) + .collect() +} + +#[test] +fn a_self_compared_record_types_as_the_self_subclass_and_carries_every_named_property() { + let (g, _) = emit_one(&record("self", serde_json::json!({}))); + let node = iri("parity-receipt", "r.json"); + assert_eq!( + g.objects(&node, RDF_TYPE)[0].as_iri(), + Some(parity("SelfComparedReceipt").as_str()) + ); + assert_eq!(objects(&g, "host"), ["gx10-a5b5"]); + assert_eq!(objects(&g, "backend"), ["cuda"]); + assert_eq!(objects(&g, "aprVersion"), ["0.65.2"]); + assert_eq!(objects(&g, "generatedAt"), ["2026-09-08"]); + assert_eq!(objects(&g, "partiallyReceipted"), ["true"]); + assert_eq!(objects(&g, "unmeasured").len(), 1); +} + +#[test] +fn a_non_self_comparator_types_as_the_oracle_subclass() { + let v = record( + "llama_cpp", + serde_json::json!({"comparator": {"kind": "llama_cpp", "comparator_sha": "39173bcac"}}), + ); + let (g, _) = emit_one(&v); + assert_eq!( + g.objects(&iri("parity-receipt", "r.json"), RDF_TYPE)[0].as_iri(), + Some(parity("OracleComparedReceipt").as_str()) + ); + let c = iri("parity-comparator", "r.json"); + assert_eq!( + g.objects(&c, &parity("comparatorSha"))[0] + .as_literal() + .map(|l| l.0), + Some("39173bcac") + ); +} + +#[test] +fn model_sha256_is_absent_when_the_record_carries_none_and_absent_is_legal() { + // ONT-4c1's rule, and the reason the shape gives model_sha256 a pattern and no minCount: six of the + // seven back-filled records never recorded a model hash, and hashing the file on the host TODAY would + // attach a claim about a different world to a receipt about 2026-09-06. + let (g, _) = emit_one(&record("self", serde_json::json!({}))); + assert!(objects(&g, "modelSha256").is_empty()); + let with = record( + "self", + serde_json::json!({"cell": {"model_sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"}}), + ); + let (g2, _) = emit_one(&with); + assert_eq!(objects(&g2, "modelSha256").len(), 1); +} + +#[test] +fn a_threshold_source_that_names_no_file_is_materialised_for_a_shape_to_refuse() { + let v = record( + "self", + serde_json::json!({"threshold_source": "evidence/parity/does-not-exist.yaml"}), + ); + let (g, stats) = emit_one(&v); + assert_eq!(stats.threshold_source_missing, 1); + assert_eq!( + objects(&g, "thresholdSourceMissing"), + ["evidence/parity/does-not-exist.yaml"] + ); +} + +#[test] +fn the_plant_removes_the_comparator_edge_every_run() { + assert!(positive_control(&record("self", serde_json::json!({})))); +} + +#[test] +fn an_unmigrated_legacy_record_is_refused_by_name_and_never_skipped() { + // The exact layout the seven records carried before #3577: no schema, metrics[] at the top level. + let dir = tempdir("unmigrated"); + let f = dir.join(EVIDENCE_DIR).join("l0-1/lambda"); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write( + f.join("legacy.json"), + r#"{"model":"./m.gguf","tokens":78,"passed":78,"failed":0,"parity":true,"metrics":[]}"#, + ) + .expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.records, 0); + assert_eq!(stats.skipped, 0, "a legacy record must not be SKIPPED"); + assert_eq!(stats.errors.len(), 1); + assert!( + stats.errors[0].what.contains("UNMIGRATED"), + "{:?}", + stats.errors[0] + ); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn a_document_that_is_not_a_parity_record_is_skipped_and_counted() { + let dir = tempdir("other"); + let f = dir.join(EVIDENCE_DIR); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write(f.join("props-abc.json"), r#"{"seed": 1, "cases": []}"#).expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!( + (stats.records, stats.skipped, stats.errors.len()), + (0, 1, 0) + ); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn the_denominator_pins_the_count_and_a_mismatch_is_reported_in_both_directions() { + let dir = tempdir("denominator"); + let f = dir.join(EVIDENCE_DIR); + std::fs::create_dir_all(&f).expect("mkdir"); + std::fs::write( + f.join("r.json"), + serde_json::to_string(&record("self", serde_json::json!({}))).expect("json"), + ) + .expect("write"); + + // Committed 2, found 1 — a record was deleted or the extractor stopped seeing it. + std::fs::write(dir.join(EXPECTED_FILE), "# count\n2\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), Some((2, 1))); + + // Committed 1, found 1 — the state the gate requires. + std::fs::write(dir.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), None); + + // Committed 0, found 1 — a receipt added without updating the denominator. THE falsifier the row names. + std::fs::write(dir.join(EXPECTED_FILE), "0\n").expect("write"); + let mut g = Graph::new(); + let stats = extract(&dir, &mut g); + assert_eq!(stats.wrong_corpus(), Some((0, 1))); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn an_absent_denominator_is_not_a_mismatch_it_is_a_different_fault() { + // "no expectation" and "a broken expectation" must not collapse into one state: the first is a missing + // declaration for the caller to report, the second is WrongCorpus. + let stats = ParityStats { + records: 3, + expected: None, + ..ParityStats::default() + }; + assert_eq!(stats.wrong_corpus(), None); +} + +#[test] +fn pointing_the_extractor_at_a_subdirectory_trips_the_denominator() { + // The row's second falsifier: a narrowed walk finds fewer records and must not read as "no violations". + let dir = tempdir("subdir"); + let deep = dir.join(EVIDENCE_DIR).join("l0-1/lambda"); + std::fs::create_dir_all(&deep).expect("mkdir"); + std::fs::write( + deep.join("r.json"), + serde_json::to_string(&record("self", serde_json::json!({}))).expect("json"), + ) + .expect("write"); + std::fs::write(dir.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g = Graph::new(); + assert_eq!(extract(&dir, &mut g).wrong_corpus(), None); + + // Same denominator, a root whose evidence/parity holds nothing: 1 expected, 0 found. + let narrow = tempdir("subdir-narrow"); + std::fs::create_dir_all(narrow.join(EVIDENCE_DIR)).expect("mkdir"); + std::fs::write(narrow.join(EXPECTED_FILE), "1\n").expect("write"); + let mut g2 = Graph::new(); + assert_eq!(extract(&narrow, &mut g2).wrong_corpus(), Some((1, 0))); + std::fs::remove_dir_all(&dir).ok(); + std::fs::remove_dir_all(&narrow).ok(); +} + +fn tempdir(tag: &str) -> PathBuf { + let p = std::env::temp_dir().join(format!( + "pmat3577-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&p).expect("tempdir"); + p +} diff --git a/crates/aprender-contracts/src/ontology/verdict.rs b/crates/aprender-contracts/src/ontology/verdict.rs index 614702615b..9e1a25db49 100644 --- a/crates/aprender-contracts/src/ontology/verdict.rs +++ b/crates/aprender-contracts/src/ontology/verdict.rs @@ -10,7 +10,7 @@ //! //! `meet = min` is Kleene's strong conjunction (K3). Exit mapping: Pass→0, Fail→1, Unknown→2 with a //! `decline: ` line. Proof obligations, all discharged by the `#[cfg(kani)]` harnesses below and -//! exhaustively by the unit tests over the 17 elements: +//! exhaustively by the unit tests over the 18 elements: //! //! - KANI-ONT-6-1 — meet laws: commutative, associative, idempotent, Pass is the identity, Fail absorbs, //! and the meet is below both operands. @@ -38,11 +38,22 @@ pub enum Reason { Advisory, ExtractorMissing, Prose, + /// PMAT-3577: an extractor RAN and read a different corpus than the tree declares — it matched a + /// different number of focus nodes than the committed denominator says, or refused a record by name. + /// Reading the wrong corpus reports the same "no violations" as reading all of it, so it is never + /// `Pass` and never a fabricated `Fail`. + /// + /// NOT named `ExtractorMiss`. [`Self::ExtractorMissing`] already means the opposite thing — an + /// extractor that does not exist — and the two would have sat one letter apart in the same lattice, + /// with the shorter a PREFIX of the longer: `grep ExtractorMiss` would match both, and any substring + /// test over the reasons would silently merge them. That is the defect class this repository keeps + /// paying for; the name says what happened instead. + WrongCorpus, } impl Reason { /// Every reason, in lattice order. - pub const ALL: [Self; 15] = [ + pub const ALL: [Self; 16] = [ Self::NotRun, Self::Skip, Self::Report, @@ -58,6 +69,7 @@ impl Reason { Self::Advisory, Self::ExtractorMissing, Self::Prose, + Self::WrongCorpus, ]; } @@ -77,7 +89,7 @@ pub enum Verdict { } impl Verdict { - /// The 17 elements: Fail, the 15 Unknowns in order, Pass. + /// The 18 elements: Fail, the 16 Unknowns in order, Pass. #[must_use] pub fn all() -> Vec { let mut v = vec![Self::Fail]; @@ -200,12 +212,12 @@ mod tests { use super::*; #[test] - fn seventeen_elements_in_lattice_order() { + fn eighteen_elements_in_lattice_order() { let all = Verdict::all(); - assert_eq!(all.len(), 17); + assert_eq!(all.len(), 18); assert!( all.windows(2).all(|w| w[0] < w[1]), - "Fail < Unknown(NotRun) < … < Unknown(Prose) < Pass" + "Fail < Unknown(NotRun) < … < Unknown(WrongCorpus) < Pass" ); } @@ -268,7 +280,7 @@ mod tests { assert_eq!(Verdict::Unknown(Reason::Skip).to_string(), "Unknown(Skip)"); let spellings: std::collections::HashSet = Verdict::all().iter().map(ToString::to_string).collect(); - assert_eq!(spellings.len(), 17, "no two elements share a spelling"); + assert_eq!(spellings.len(), 18, "no two elements share a spelling"); assert_eq!( serde_json::to_string(&Verdict::Unknown(Reason::NotArmed)) .ok() diff --git a/crates/aprender-contracts/src/proof_status_tests.rs b/crates/aprender-contracts/src/proof_status_tests.rs index dac11d4b24..508e583760 100644 --- a/crates/aprender-contracts/src/proof_status_tests.rs +++ b/crates/aprender-contracts/src/proof_status_tests.rs @@ -1,5 +1,5 @@ use crate::obligation_matrix::{ - format_obligation_table, obligation_matrix, property_words_match, truncate, + format_obligation_table, obligation_matrix, property_words_match, truncate, L2Status, }; use crate::proof_status::*; use crate::schema::{parse_contract_str, Contract}; @@ -355,6 +355,56 @@ fn truncate_helper() { assert_eq!(truncate("hello world", 5), "hello"); } +/// #3338: `pv proof-status contracts/ --table` PANICKED on the real corpus. +/// +/// `truncate` sliced by BYTE index, and the width is `min(max byte len, 40)`. +/// Eight contracts hold an obligation property whose byte 40 lands inside a +/// multi-byte char; the first one walked is +/// `contracts/apr-inspect-quantization-v1.yaml`: +/// +/// ```text +/// byte index 40 is not a char boundary; it is inside '∈' (bytes 39..42) +/// of `for Q4_K_M Qwen2.5-Coder, quantization ∈ {Q4_K, Q6_K}` +/// ``` +/// +/// The fixture IS that property, so the cut is inside the same char. +#[test] +fn truncate_cuts_on_a_char_boundary_not_a_byte() { + let s = "for Q4_K_M Qwen2.5-Coder, quantization ∈ {Q4_K, Q6_K}"; + assert!( + !s.is_char_boundary(40), + "fixture must cut INSIDE a multi-byte char, else it proves nothing" + ); + let t = truncate(s, 40); + assert!(s.starts_with(t), "truncation must be a prefix"); + assert!(t.len() <= 40, "truncation must not exceed the budget"); + assert_eq!(t, "for Q4_K_M Qwen2.5-Coder, quantization "); +} + +/// The panic reached the operator through `format_obligation_table`, so the +/// table path gets its own row rather than only the helper. +#[test] +fn format_obligation_table_survives_a_multibyte_property() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "Multi-byte property at the cut" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - type: invariant + property: "for Q4_K_M Qwen2.5-Coder, quantization ∈ {Q4_K, Q6_K}" +falsification_tests: [] +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let matrices = obligation_matrix(&[("multibyte-v1".to_string(), &c)]); + let text = format_obligation_table(&matrices); + assert!(text.contains("Contract: multibyte-v1")); +} + #[test] fn schema_version_present() { let report = proof_status_report(&[], None, false); @@ -376,33 +426,236 @@ fn obligation_matrix_empty() { assert!(matrices.is_empty()); } +/// #3347: this test used to be named `obligation_matrix_index_based_l2` and +/// asserted the defect -- 3 obligations and 3 tests, none of them linked to +/// anything, scored L2 across the board because `idx < 3`. +/// +/// The contract says nothing about which test covers which obligation, so the +/// honest verdict is Unknown and the level stays L1. #[test] -fn obligation_matrix_index_based_l2() { - // 3 obligations, 3 tests, 0 kani => all L2 by index +fn obligation_matrix_unlinked_tests_are_unknown_not_l2() { let c = minimal_contract(3, 3, 0); let matrices = obligation_matrix(&[("test-v1".to_string(), &c)]); assert_eq!(matrices.len(), 1); assert_eq!(matrices[0].obligations.len(), 3); for ob in &matrices[0].obligations { - assert!(ob.l2_tested); + assert_eq!(ob.l2, L2Status::Unknown); + assert!(!ob.l2.is_tested()); assert!(!ob.l3_kani); assert!(!ob.l4_lean); - assert_eq!(ob.max_level, ProofLevel::L2); + assert_eq!(ob.max_level, ProofLevel::L1); } } +/// Zero falsification tests is a READING, not an unread window: no test +/// exists, so no test covers this obligation. Untested, not Unknown. #[test] -fn obligation_matrix_no_tests_is_l1() { - // 2 obligations, 0 tests, 0 kani => L1 +fn obligation_matrix_no_tests_is_untested_and_l1() { let c = minimal_contract(2, 0, 0); let matrices = obligation_matrix(&[("test-v1".to_string(), &c)]); assert_eq!(matrices[0].obligations.len(), 2); for ob in &matrices[0].obligations { - assert!(!ob.l2_tested); + assert_eq!(ob.l2, L2Status::Untested); assert_eq!(ob.max_level, ProofLevel::L1); } } +/// `proof_obligations[].discharged_by` -- the link written from the +/// obligation's side, and the most-used spelling in `contracts/` (89). +#[test] +fn discharged_by_links_an_obligation_to_its_test() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "discharged_by, both resolvable shapes" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - type: invariant + property: "By index" + discharged_by: falsification_tests[0] + - type: invariant + property: "By test id" + discharged_by: FT-002 + - type: invariant + property: "Out of range" + discharged_by: falsification_tests[9] + - type: invariant + property: "Names a kani harness, not a test" + discharged_by: KANI-X-001 +falsification_tests: + - id: FT-001 + rule: "r" + prediction: "p" + if_fails: "f" + - id: FT-002 + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let obs = &obligation_matrix(&[("db-v1".to_string(), &c)])[0].obligations; + assert_eq!(obs[0].l2, L2Status::Tested, "falsification_tests[0] exists"); + assert_eq!(obs[1].l2, L2Status::Tested, "FT-002 exists"); + assert_eq!( + obs[2].l2, + L2Status::Untested, + "falsification_tests[9] is out of range over 2 tests -- a dangling \ + citation is not a proof" + ); + assert_eq!( + obs[3].l2, + L2Status::Untested, + "a kani harness id is the L3 column, never an L2 link" + ); +} + +/// `binds_to:` is the second spelling of `falsification_tests[].obligation` +/// (38 entries vs 26). It is a serde alias, which is safe ONLY because no +/// entry in `contracts/` carries both keys -- serde would make that a +/// `duplicate field` parse error rather than a silent pick. +#[test] +fn binds_to_is_the_same_link_as_obligation() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "binds_to alias" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - id: OB-1 + type: invariant + property: "Bound by alias" + - id: OB-2 + type: invariant + property: "Bound by nothing" +falsification_tests: + - id: FT-001 + binds_to: OB-1 + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let cited: Vec<&str> = c.falsification_tests[0] + .obligation + .as_ref() + .expect("binds_to must land on the obligation field, not be dropped") + .targets() + .collect(); + assert_eq!(cited, vec!["OB-1"]); + let obs = &obligation_matrix(&[("alias-v1".to_string(), &c)])[0].obligations; + assert_eq!(obs[0].l2, L2Status::Tested); + assert_eq!(obs[1].l2, L2Status::Untested); +} + +/// A contract whose every link DANGLES was not read -- reporting its +/// obligations as Untested would be inventing a finding out of a parse +/// failure. `apr-code-harness-ir-v1` is the real instance: 8 tests cite +/// `OBLIG-IR-N` and that contract's obligations carry no `id` at all. +#[test] +fn links_that_all_dangle_are_unknown_not_untested() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "every citation dangles" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - type: invariant + property: "Alpha" + - type: invariant + property: "Beta" +falsification_tests: + - id: FT-001 + obligation: OBLIG-NOBODY-1 + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let obs = &obligation_matrix(&[("dangle-v1".to_string(), &c)])[0].obligations; + for ob in obs { + assert_eq!(ob.l2, L2Status::Unknown); + } +} + +/// One test may discharge several obligations in one comma-separated field. +#[test] +fn a_comma_separated_citation_names_several_obligations() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "comma list" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - id: OB-1 + type: invariant + property: "One" + - id: OB-2 + type: invariant + property: "Two" + - id: OB-3 + type: invariant + property: "Three" +falsification_tests: + - id: FT-001 + obligation: "OB-1, OB-3" + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let obs = &obligation_matrix(&[("comma-v1".to_string(), &c)])[0].obligations; + assert_eq!(obs[0].l2, L2Status::Tested); + assert_eq!(obs[1].l2, L2Status::Untested); + assert_eq!(obs[2].l2, L2Status::Tested); +} + +/// A citation may also name the obligation by its exact `property` text -- +/// `apr-mcp-stdio-drain-v1` binds all six of its tests that way. +#[test] +fn a_citation_may_name_the_property_text() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "property-text citation" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - type: invariant + property: "drain-on-every-exit" + - type: invariant + property: "no-false-error" +falsification_tests: + - id: FT-001 + obligation: drain-on-every-exit + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let obs = &obligation_matrix(&[("prop-v1".to_string(), &c)])[0].obligations; + assert_eq!(obs[0].l2, L2Status::Tested); + assert_eq!(obs[1].l2, L2Status::Untested); +} + #[test] fn obligation_matrix_lean_proved() { // Build a contract with a Lean-proved obligation @@ -442,19 +695,21 @@ kani_harnesses: let matrices = obligation_matrix(&[("test-v1".to_string(), &c)]); assert_eq!(matrices[0].obligations.len(), 2); - // First obligation: L2 (index), L3 (kani property match "sums"), L4 (lean proved) + // First obligation: L3 (kani property match "sums"), L4 (lean proved). Its + // L2 is Unknown -- the two tests name no obligation (#3347) -- which does + // not disturb a level earned higher up the ladder. let ob0 = &matrices[0].obligations[0]; - assert!(ob0.l2_tested); + assert_eq!(ob0.l2, L2Status::Unknown); assert!(ob0.l3_kani); assert!(ob0.l4_lean); assert_eq!(ob0.max_level, ProofLevel::L4); - // Second obligation "Range is strictly positive": L2 (index), no kani match, no lean + // Second obligation "Range is strictly positive": nothing at all. let ob1 = &matrices[0].obligations[1]; - assert!(ob1.l2_tested); + assert_eq!(ob1.l2, L2Status::Unknown); assert!(!ob1.l3_kani); assert!(!ob1.l4_lean); - assert_eq!(ob1.max_level, ProofLevel::L2); + assert_eq!(ob1.max_level, ProofLevel::L1); } #[test] @@ -484,7 +739,78 @@ kani_harnesses: [] let matrices = obligation_matrix(&[("test-v1".to_string(), &c)]); let ob = &matrices[0].obligations[0]; assert!(!ob.l4_lean); - assert_eq!(ob.max_level, ProofLevel::L2); + // The lone test names no obligation, so L2 is Unknown and the level is L1. + assert_eq!(ob.l2, L2Status::Unknown); + assert_eq!(ob.max_level, ProofLevel::L1); +} + +/// RED FIRST (#3347). The L2 column ticked on `idx < falsification_tests.len()` +/// — a COUNT, not a link — so every obligation of a contract with enough tests +/// showed ✓ whoever those tests were about. +/// +/// This fixture makes the two disagree: TWO obligations, TWO tests, and BOTH +/// tests cite `OB-A`. Nothing anywhere claims `OB-B` is tested. Under the index +/// rule `OB-B` is index 1 < 2 tests, so it ticked. +/// +/// The assertion goes through `format_obligation_table` deliberately: the +/// rendered L2 cell is the surface the operator reads, and it is the surface +/// that was lying. It is also API-stable, so this row is the SAME test before +/// and after the fix — it fails on the old code and passes on the new. +#[test] +fn l2_does_not_tick_for_an_obligation_no_test_cites() { + let yaml = r#" +metadata: + version: "1.0.0" + description: "Two obligations, two tests, both tests cite OB-A" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - id: OB-A + type: invariant + property: "Alpha holds" + - id: OB-B + type: invariant + property: "Beta holds" +falsification_tests: + - id: FT-001 + obligation: OB-A + rule: "alpha one" + prediction: "p" + if_fails: "f" + - id: FT-002 + obligation: OB-A + rule: "alpha two" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let c = parse_contract_str(yaml).unwrap(); + let matrices = obligation_matrix(&[("cross-cited-v1".to_string(), &c)]); + let text = format_obligation_table(&matrices); + + let row_a = table_row(&text, "Alpha holds"); + let row_b = table_row(&text, "Beta holds"); + + // Discrimination: the contract DOES bind OB-A, so a fix that simply stopped + // ticking everything would fail here. + assert!( + row_a.contains('\u{2713}'), + "OB-A is cited by two tests and must stay ticked\nrow: {row_a}" + ); + assert!( + !row_b.contains('\u{2713}'), + "OB-B is cited by NO test — the L2 column ticked it from a count, not a link\nrow: {row_b}" + ); +} + +/// Pull one obligation's rendered row out of the table by its property text. +fn table_row<'a>(table: &'a str, property: &str) -> &'a str { + table + .lines() + .find(|l| l.contains(property)) + .unwrap_or_else(|| panic!("no table row for property `{property}`:\n{table}")) } #[test] @@ -521,14 +847,55 @@ fn format_obligation_table_header() { assert!(text.contains("Status")); } +/// All three L2 cells are reachable from the renderer, and they are distinct +/// glyphs. A fix that collapsed Unknown onto either tick or cross would fail +/// here rather than pass quietly. #[test] -fn format_obligation_table_check_marks() { - let c = minimal_contract(1, 1, 0); - let matrices = obligation_matrix(&[("test-v1".to_string(), &c)]); - let text = format_obligation_table(&matrices); - // L2 should be checked, L3/L4 should be crossed - assert!(text.contains('\u{2713}')); // check mark - assert!(text.contains('\u{2717}')); // cross mark +fn format_obligation_table_renders_all_three_l2_cells() { + // 1 obligation, 1 test, no link => `?`; L3/L4 crossed. + let unknown = minimal_contract(1, 1, 0); + let text = format_obligation_table(&obligation_matrix(&[("u-v1".to_string(), &unknown)])); + assert!(text.contains('?'), "unlinked L2 renders as `?`:\n{text}"); + assert!( + text.contains('\u{2717}'), + "L3/L4 render as crosses:\n{text}" + ); + + // 0 tests => a cross in the L2 column, not a `?`. + let untested = minimal_contract(1, 0, 0); + let text = format_obligation_table(&obligation_matrix(&[("x-v1".to_string(), &untested)])); + assert!( + !text.contains('?'), + "0 tests is a reading, not Unknown:\n{text}" + ); + + // A linked obligation still ticks. + let yaml = r#" +metadata: + version: "1.0.0" + description: "linked" + references: ["Paper"] +equations: + f: + formula: "f(x) = x" +proof_obligations: + - id: OB-1 + type: invariant + property: "Linked prop" +falsification_tests: + - id: FT-001 + obligation: OB-1 + rule: "r" + prediction: "p" + if_fails: "f" +kani_harnesses: [] +"#; + let linked = parse_contract_str(yaml).unwrap(); + let text = format_obligation_table(&obligation_matrix(&[("l-v1".to_string(), &linked)])); + assert!( + text.contains('\u{2713}'), + "a linked obligation ticks:\n{text}" + ); } #[test] diff --git a/crates/aprender-contracts/src/schema/crux_intake_tests.rs b/crates/aprender-contracts/src/schema/crux_intake_tests.rs index 4a6728c509..ed3250c1fa 100644 --- a/crates/aprender-contracts/src/schema/crux_intake_tests.rs +++ b/crates/aprender-contracts/src/schema/crux_intake_tests.rs @@ -216,6 +216,8 @@ fn competitor_registry_covers_the_corpus_vocabulary() { // 17 contracts: burn ×7, linfa ×10. "burn", "linfa", + // Category O — AutoML Parity (aprender#3370, 2026-09-16). 24 contracts. + "autogluon", ] { assert!( CRUX_COMPETITORS.contains(&required), @@ -258,6 +260,8 @@ fn beat_incumbents_cannot_name_the_crux_corpus() { // is why category N required a registry edit rather than a reuse. "burn", "linfa", + // AutoGluon: not a BEAT pillar either; scikit-learn stays the pillar. + "autogluon", ] { assert!(!beat_accepts(c), "BEAT_INCUMBENTS unexpectedly accepts {c}"); } diff --git a/crates/aprender-contracts/src/schema/types.rs b/crates/aprender-contracts/src/schema/types.rs index fc6481fd51..aff241e899 100644 --- a/crates/aprender-contracts/src/schema/types.rs +++ b/crates/aprender-contracts/src/schema/types.rs @@ -531,6 +531,40 @@ pub struct Equation { pub guarantees: Option, } +/// One or more cited targets (#3347). +/// +/// The corpus writes an obligation-to-test citation three ways and all three +/// are authored by hand, so the type accepts all three rather than making one +/// of them a parse error: a scalar (`FALSIFY-PM-004`), a comma-separated +/// scalar (`apr-serve-cancellation-v1` names four in one field), and a YAML +/// sequence (`publish-manifest-v1`, the only one today -- and the one that +/// proved `Option` was the wrong type by failing the WHOLE corpus +/// with `invalid type: sequence, expected a string`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Citation { + /// A single field, possibly holding a comma-separated list. + One(String), + /// A YAML sequence of targets. + Many(Vec), +} + +impl Citation { + /// The cited targets, trimmed, with empties dropped. A comma splits a + /// scalar because contracts write lists both ways. + pub fn targets(&self) -> impl Iterator { + let slice: &[String] = match self { + Self::One(s) => std::slice::from_ref(s), + Self::Many(v) => v.as_slice(), + }; + slice + .iter() + .flat_map(|s| s.split(',')) + .map(str::trim) + .filter(|t| !t.is_empty()) + } +} + /// A proof obligation derived from an equation. /// /// 26 obligation types: 19 property types plus 7 Design by Contract @@ -571,6 +605,17 @@ pub struct ProofObligation { pub tolerance: Option, #[serde(default)] pub applies_to: Option, + /// The falsification test(s) that discharge this obligation (#3347) -- + /// the same link as `FalsificationTest::obligation`, written from the + /// obligation's side. 89 obligations in `contracts/` carry it and it is + /// the most-used spelling of the link; it too was dropped on parse. + /// + /// Two resolvable shapes, both measured: `falsification_tests[N]` (62, + /// resolves only when `N` is in range) and a test `id` (13). The + /// remaining 14 are comma-separated id lists, prose, or a KANI harness id + /// -- a kani id is NOT an L2 link and deliberately does not resolve. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub discharged_by: Option, /// Why this obligation is NOT a property of code (PMAT-3091) -- e.g. a /// checkpoint fact, an `O()` with no constant, a throughput claim. /// @@ -783,6 +828,24 @@ pub struct FalsificationTest { /// Defaulted because several legacy diagnostic contracts omit it. #[serde(default, alias = "fails_if")] pub if_fails: String, + /// The obligation this test discharges — the only machine-readable claim + /// that THIS test proves THAT obligation (#3347). + /// + /// Until this field existed the key was written to disk and silently + /// dropped on parse (the same shape as `id` in #3314 and `test_harness` + /// in #2465: no `deny_unknown_fields`, so serde discarded it), which is + /// why `obligation_matrix` had nothing to read and fell back to comparing + /// an INDEX against `falsification_tests.len()`. + /// + /// Measured over `contracts/` (1,842 files, 4,691 falsification tests): + /// 26 entries spell it `obligation:` and 38 spell it `binds_to:`. No entry + /// carries BOTH — checked, and it matters, because serde collapses an + /// alias pair present on one mapping into a `duplicate field` parse error. + /// + /// Resolved against the obligation's `id`, then its exact `property` text. + /// A comma-separated list cites several obligations. + #[serde(default, alias = "binds_to", skip_serializing_if = "Option::is_none")] + pub obligation: Option, } /// A Kani bounded model checking harness definition. diff --git a/crates/aprender-contracts/src/schema/validator.rs b/crates/aprender-contracts/src/schema/validator.rs index 32d97d4b83..83dc8303e9 100644 --- a/crates/aprender-contracts/src/schema/validator.rs +++ b/crates/aprender-contracts/src/schema/validator.rs @@ -120,8 +120,18 @@ fn explain_kind_default(caused: &mut [Violation]) { /// exercised by at least one contract in `contracts/`; adding a competitor is a /// deliberate one-line edit here plus a test, which is the point — an open /// domain is what let `THIS-COMPETITOR-DOES-NOT-EXIST` validate. -pub(crate) const CRUX_COMPETITORS: [&str; 14] = [ +pub(crate) const CRUX_COMPETITORS: [&str; 15] = [ "apr-qa-playbook", + // AutoGluon (autogluon/autogluon) — the AutoML library, 1.6.3 at admission + // (../autogluon @ 77946149). Added 2026-09-16 with 24 category-O stories + // extracted from its three predictors: TabularPredictor (fit(label), + // presets, leaderboard, feature pipeline, bagging/stacking/weighted + // ensemble, budgets, deployment) and TimeSeriesPredictor (panel data, + // quantile metrics, backtesting, local baselines, Chronos-2 class + // pretrained forecasters). NOT a BEAT pillar — aprender claims no pinned + // benchmark win over AutoGluon; this is a capability/UX source. Epic + // aprender#3370. + "autogluon", // Burn (tracel-ai/burn) — the Rust deep-learning framework, 0.21.0 / 15.9k // stars / 312 reverse-dependencies at admission. Added 2026-09-12 with 7 // category-N stories extracted from its crate surface: burn-linalg (SVD), diff --git a/crates/aprender-core/build_codegen.rs b/crates/aprender-core/build_codegen.rs index 16c506f1c9..114fc9a3d5 100644 --- a/crates/aprender-core/build_codegen.rs +++ b/crates/aprender-core/build_codegen.rs @@ -204,6 +204,7 @@ fn generate_family_registration(f: &FamilyData) -> String { \x20 positional_encoding: PositionalEncoding::from_str_contract(\"{}\").unwrap_or(PositionalEncoding::Rope),\n\ \x20 mlp_type: MlpType::from_str_contract(\"{}\").unwrap_or(MlpType::SwiGlu),\n\ \x20 qk_norm: {},\n\ + \x20 deltanet: {},\n\ \x20 }},\n\ \x20 tensor_template: TensorTemplate {{\n\ \x20 embedding: \"{}\".to_string(),\n\ @@ -243,6 +244,7 @@ fn generate_family_registration(f: &FamilyData) -> String { f.constraints.position, f.constraints.mlp, f.constraints.qk_norm, + deltanet_expr(f), f.embedding_tensor, f.lm_head_tensor .as_ref() @@ -490,3 +492,16 @@ fn generate_algebraic_proofs(f: &FamilyData) -> String { out.push('\n'); out } + +/// #3346: render a family's Gated DeltaNet shape as a Rust expression, so the +/// compiled-in registry carries the same keys the runtime YAML parser does. +/// A family that declares none renders `None` and keeps dense accounting. +fn deltanet_expr(f: &FamilyData) -> String { + match &f.constraints.deltanet { + None => "None".to_string(), + Some(d) => format!( + "Some(DeltaNetShape {{ inner_size: {}, state_size: {}, conv_kernel: {}, group_count: {}, full_attention_interval: {} }})", + d.inner_size, d.state_size, d.conv_kernel, d.group_count, d.full_attention_interval + ), + } +} diff --git a/crates/aprender-core/build_parsing.rs b/crates/aprender-core/build_parsing.rs index c0cedbca33..724f327396 100644 --- a/crates/aprender-core/build_parsing.rs +++ b/crates/aprender-core/build_parsing.rs @@ -52,6 +52,34 @@ struct ConstraintsData { position: String, mlp: String, qk_norm: bool, + /// #3346: Gated DeltaNet shape keys. `None` unless the descriptor declares + /// both `inner_size` and `state_size`. + deltanet: Option, +} + +/// #3346: the `inner_size`/`state_size`/`conv_kernel`/`group_count` + +/// `full_attention_interval` block of a hybrid family's `constraints:`. +struct DeltaNetData { + inner_size: usize, + state_size: usize, + conv_kernel: usize, + group_count: usize, + full_attention_interval: usize, +} + +/// #3346: read the Gated DeltaNet shape out of a `constraints:` section. +/// Both `inner_size` and `state_size` are required — they are what make the +/// block a DeltaNet mixer — so every other family yields `None`. +fn parse_deltanet_data(section: &str) -> Option { + let inner_size = get_usize(section, "inner_size")?; + let state_size = get_usize(section, "state_size")?; + Some(DeltaNetData { + inner_size, + state_size, + conv_kernel: get_usize(section, "conv_kernel").unwrap_or(0), + group_count: get_usize(section, "group_count").unwrap_or(0), + full_attention_interval: get_usize(section, "full_attention_interval").unwrap_or(0), + }) } // ============================================================================ @@ -144,6 +172,7 @@ fn parse_family_yaml(content: &str, path: &Path) -> FamilyData { position: c_str("positional_encoding", "rope"), mlp: c_str("mlp_type", "swiglu"), qk_norm: get_bool(&constraints_section, "qk_norm").unwrap_or(false), + deltanet: parse_deltanet_data(&constraints_section), }; // Parse tensor_template diff --git a/crates/aprender-core/src/format/gguf/builder.rs b/crates/aprender-core/src/format/gguf/builder.rs index 01ab5ad0b5..5aca8570aa 100644 --- a/crates/aprender-core/src/format/gguf/builder.rs +++ b/crates/aprender-core/src/format/gguf/builder.rs @@ -15,7 +15,7 @@ fn test_from_bytes_zero_tensors_zero_metadata() { // ======================================================================== /// Build a complete synthetic GGUF file with one F32 tensor and optional metadata -fn build_synthetic_gguf_with_tensor( +pub(super) fn build_synthetic_gguf_with_tensor( tensor_name: &str, dims: &[u64], dtype: u32, diff --git a/crates/aprender-core/src/format/gguf/dequant.rs b/crates/aprender-core/src/format/gguf/dequant.rs index e01794f07d..b480c94776 100644 --- a/crates/aprender-core/src/format/gguf/dequant.rs +++ b/crates/aprender-core/src/format/gguf/dequant.rs @@ -2,8 +2,9 @@ //! //! K-quant formats (Q4_K, Q5_K, Q6_K) delegate to `trueno_quant` — the single //! source of truth in the Sovereign AI Stack. Legacy GGML formats (Q4_0, Q8_0, -//! Q5_0, Q5_1, Q4_1, Q2_K, Q3_K, IQ*) have no trueno equivalent and keep their -//! inline implementations. +//! Q5_0, Q5_1, Q4_1, Q2_K, Q3_K) have no trueno equivalent and keep their +//! inline implementations. IQ* has no dequantizer here: `get_tensor_f32` refuses +//! it (#3656) rather than approximate it. use crate::error::{AprenderError, Result}; use provable_contracts_macros::ensures; diff --git a/crates/aprender-core/src/format/gguf/dequant_f16_tests.rs b/crates/aprender-core/src/format/gguf/dequant_f16_tests.rs index bec970c006..013b6c189d 100644 --- a/crates/aprender-core/src/format/gguf/dequant_f16_tests.rs +++ b/crates/aprender-core/src/format/gguf/dequant_f16_tests.rs @@ -390,49 +390,6 @@ mod tests { ); } - // ========================================================================= - // Dequantize IQ approximate - // ========================================================================= - - #[test] - fn test_dequantize_iq_approximate_iq2() { - let data = vec![128u8; 1024]; - let result = dequantize_iq_approximate(&data, 0, 64, 13); // IQ2_XXS - assert_eq!(result.len(), 64); - } - - #[test] - fn test_dequantize_iq_approximate_iq3() { - let data = vec![128u8; 1024]; - let result = dequantize_iq_approximate(&data, 0, 64, 16); // IQ3_XXS - assert_eq!(result.len(), 64); - } - - #[test] - fn test_dequantize_iq_approximate_iq1() { - let data = vec![128u8; 1024]; - let result = dequantize_iq_approximate(&data, 0, 64, 18); // IQ1_S - assert_eq!(result.len(), 64); - } - - #[test] - fn test_dequantize_iq_approximate_default_dtype() { - let data = vec![128u8; 1024]; - let result = dequantize_iq_approximate(&data, 0, 64, 99); // Unknown dtype - assert_eq!(result.len(), 64); - } - - #[test] - fn test_dequantize_iq_approximate_byte_out_of_range() { - // Small data so byte_idx exceeds data.len() - let data = vec![128u8; 4]; - let result = dequantize_iq_approximate(&data, 0, 256, 13); - // Should still produce 256 elements (some will be 0.0 for out-of-range bytes) - assert_eq!(result.len(), 256); - // Verify some elements are 0.0 (from the byte_idx >= data.len() path) - assert!(result.iter().any(|&v| v == 0.0)); - } - #[test] fn test_dequantize_q4_0_with_nonzero_start() { // 18 bytes padding + 18 bytes block; put scale at offset 18 diff --git a/crates/aprender-core/src/format/gguf/dequantize.rs b/crates/aprender-core/src/format/gguf/dequantize.rs index ea06bcf852..674a19a683 100644 --- a/crates/aprender-core/src/format/gguf/dequantize.rs +++ b/crates/aprender-core/src/format/gguf/dequantize.rs @@ -243,55 +243,3 @@ pub(crate) fn dequantize_q3_k(data: &[u8], start: usize, num_elements: usize) -> result.truncate(num_elements); Ok(result) } - -/// Approximate dequantization for I-quants (IQ2, IQ3, IQ4) -/// These use importance-weighted quantization with lookup tables. -/// For import purposes, we approximate with a simple linear mapping. -pub(crate) fn dequantize_iq_approximate( - data: &[u8], - start: usize, - num_elements: usize, - dtype: u32, -) -> Vec { - // I-quants have variable block sizes and complex lookup tables - // Approximate by treating as low-bit quantization with estimated scale - - let (bits_per_element, block_size): (usize, usize) = match dtype { - 13..=15 => (2, 256), // IQ2_XXS, IQ2_XS, IQ2_S - 16 | 17 => (3, 256), // IQ3_XXS, IQ3_S - 18 => (1, 256), // IQ1_S - _ => (4, 256), // IQ4_NL, IQ4_XS, and default - }; - - let bytes_per_block = (block_size * bits_per_element).div_ceil(8) + 4; // data + scale overhead - let num_blocks = num_elements.div_ceil(block_size); - - // For approximation, create small random-ish values based on byte patterns - // This is NOT correct dequantization but allows import to proceed - let mut result = Vec::with_capacity(num_elements); - let scale = 0.01; // Small scale for approximate values - - for block_idx in 0..num_blocks { - let block_start = start + block_idx * bytes_per_block; - - for i in 0..block_size { - if result.len() >= num_elements { - break; - } - - // Use byte pattern to generate approximate value - let byte_idx = block_start + (i * bits_per_element) / 8; - if byte_idx < data.len() { - let byte_val = data[byte_idx]; - // Map to roughly centered distribution - let approx = (f32::from(byte_val) - 128.0) * scale; - result.push(approx); - } else { - result.push(0.0); - } - } - } - - result.truncate(num_elements); - result -} diff --git a/crates/aprender-core/src/format/gguf/reader.rs b/crates/aprender-core/src/format/gguf/reader.rs index 03ad1957ec..198931c5a5 100644 --- a/crates/aprender-core/src/format/gguf/reader.rs +++ b/crates/aprender-core/src/format/gguf/reader.rs @@ -21,8 +21,8 @@ const MAX_DIMS: u32 = 16; const MAX_TENSOR_ELEMENTS: usize = 4_000_000_000; use super::dequant::{ - dequantize_iq_approximate, dequantize_q2_k, dequantize_q3_k, dequantize_q4_k, dequantize_q5_1, - dequantize_q5_k, dequantize_q6_k, f16_to_f32, + dequantize_q2_k, dequantize_q3_k, dequantize_q4_k, dequantize_q5_1, dequantize_q5_k, + dequantize_q6_k, f16_to_f32, }; use super::types::{ padding_for_alignment, GgufValue, TensorDataMap, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, diff --git a/crates/aprender-core/src/format/gguf/reader_tests.rs b/crates/aprender-core/src/format/gguf/reader_tests.rs index 2153d79826..e35a250c2a 100644 --- a/crates/aprender-core/src/format/gguf/reader_tests.rs +++ b/crates/aprender-core/src/format/gguf/reader_tests.rs @@ -449,3 +449,5 @@ fn test_from_bytes_invalid_magic() { mod builder; #[path = "reader_tests_accessors.rs"] mod reader_tests_accessors; +#[path = "reader_tests_ggml_sizes.rs"] +mod reader_tests_ggml_sizes; diff --git a/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs new file mode 100644 index 0000000000..908d4df2b0 --- /dev/null +++ b/crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs @@ -0,0 +1,201 @@ +//! #3601: `get_tensor_raw` sizes every live ggml type — and the two commands that read +//! those sizes (`apr inspect`, `apr tensors`) accept an IQ/TQ GGUF instead of calling it +//! corrupt. And #3656: sized is not readable — `get_tensor_f32` refuses what it has no +//! dequantizer for instead of approximating it. +//! +//! The sizes below are typed from ggml-common.h's block structs on purpose, NOT read +//! back from `trueno_quant::TRAITS`: a test that asked the table what the table says +//! could not fail. `TRAITS` has its own upstream-fixture test (PMAT-3430); these pin +//! what this reader does with it. + +use super::builder::build_synthetic_gguf_with_tensor; +use super::*; + +/// (ggml id, name, elements per block, bytes per block) — every IQ*/TQ* type, which is +/// every id the pre-#3601 match refused although ggml defines a layout for it. +const IQ_AND_TQ: [(u32, &str, usize, usize); 11] = [ + (16, "IQ2_XXS", 256, 66), // d + qs[QK_K/8] u16 + (17, "IQ2_XS", 256, 74), // d + qs[QK_K/8] u16 + scales[QK_K/32] + (18, "IQ3_XXS", 256, 98), // d + qs[3*QK_K/8] + (19, "IQ1_S", 256, 50), // d + qs[QK_K/8] + qh[QK_K/32] u16 + (20, "IQ4_NL", 32, 18), // d + qs[QK4_NL/2] + (21, "IQ3_S", 256, 110), // d + qs[QK_K/4] + qh[QK_K/32] + signs[QK_K/8] + scales[4] + (22, "IQ2_S", 256, 82), // d + qs[QK_K/4] + qh[QK_K/32] + scales[QK_K/32] + (23, "IQ4_XS", 256, 136), // d + scales_h u16 + scales_l[QK_K/64] + qs[QK_K/2] + (29, "IQ1_M", 256, 56), // qs[QK_K/8] + qh[QK_K/16] + scales[QK_K/32] + (34, "TQ1_0", 256, 54), // qs[(QK_K - 4*QK_K/64)/5] + qh[QK_K/64] + d + (35, "TQ2_0", 256, 66), // qs[QK_K/4] + d +]; + +/// A reader over one tensor of `dtype` with `dims`, whose data section holds `payload` +/// followed by `trailing` bytes of 0xEE — so a reader that sizes too large reads 0xEE, +/// and one that sizes too small returns fewer bytes than the payload. +fn one_tensor(dims: &[u64], dtype: u32, payload: &[u8], trailing: usize) -> GgufReader { + let mut data = payload.to_vec(); + data.extend(std::iter::repeat_n(0xEE_u8, trailing)); + let bytes = build_synthetic_gguf_with_tensor("t.weight", dims, dtype, &data, &[]); + GgufReader::from_bytes(bytes).expect("synthetic GGUF parses") +} + +/// A payload whose bytes are all distinct from the 0xEE tail. +fn payload(len: usize) -> Vec { + (0..len).map(|i| (i % 200) as u8).collect() +} + +/// done_when 2: every IQ/TQ id is sized, at exactly two blocks' worth of bytes. +#[test] +fn test_3601_get_tensor_raw_sizes_every_iq_and_tq_type() { + for (id, name, blck, type_size) in IQ_AND_TQ { + let want = payload(2 * type_size); + let reader = one_tensor(&[2 * blck as u64], id, &want, 64); + let (bytes, shape, dtype) = reader + .get_tensor_raw("t.weight") + .unwrap_or_else(|e| panic!("{name} (id {id}) must be sized, got: {e}")); + assert_eq!(dtype, id, "{name}: dtype id must round-trip"); + assert_eq!(shape, vec![2 * blck], "{name}: shape"); + assert_eq!(bytes, want, "{name}: exactly two blocks, no tail byte"); + } +} + +/// A 2-D tensor is sized from its full element count: 3 rows of one block each. +#[test] +fn test_3601_two_d_iq4_xs_tensor_is_sized_over_all_rows() { + let want = payload(3 * 136); + let reader = one_tensor(&[256, 3], 23, &want, 64); + let (bytes, _, _) = reader.get_tensor_raw("t.weight").expect("IQ4_XS sized"); + assert_eq!(bytes, want); +} + +/// Behaviour preservation: the 15 ids the hand-typed match sized keep their byte count +/// for every element count, including counts that are not a whole number of blocks +/// (the arms rounded DOWN, and so does the table-driven path). +#[test] +fn test_3601_ids_sized_before_keep_their_byte_counts() { + // The pre-#3601 arms, transcribed: (id, divisor, multiplier). + const PRE_3601: [(u32, usize, usize); 15] = [ + (0, 1, 4), + (1, 1, 2), + (2, 32, 18), + (3, 32, 20), + (6, 32, 22), + (7, 32, 24), + (8, 32, 34), + (9, 32, 36), + (10, 256, 84), + (11, 256, 110), + (12, 256, 144), + (13, 256, 176), + (14, 256, 210), + (15, 256, 292), + (30, 1, 2), + ]; + for (id, div, mul) in PRE_3601 { + for n in [1, div.saturating_sub(1).max(1), div, div + 1, 2 * div, 1000] { + let expected = (n / div) * mul; + let reader = one_tensor(&[n as u64], id, &payload(expected), 512); + let (bytes, _, _) = reader + .get_tensor_raw("t.weight") + .unwrap_or_else(|e| panic!("id {id} at {n} elements: {e}")); + assert_eq!(bytes.len(), expected, "id {id} at {n} elements"); + } + } +} + +/// done_when 3, removed id: refused by NAME, as a removed type — not as corruption, and +/// not as the bare "Unsupported dtype N" it used to be. +#[test] +fn test_3601_removed_id_is_refused_by_name() { + let reader = one_tensor(&[64], 32, &payload(64), 0); + let err = reader + .get_tensor_raw("t.weight") + .expect_err("Q4_0_4_8 has no layout upstream") + .to_string(); + for needle in ["t.weight", "32", "Q4_0_4_8", "removed upstream", "ggml-type-v1.yaml"] { + assert!(err.contains(needle), "missing {needle:?} in: {err}"); + } + assert!(!err.contains("Unsupported dtype"), "bare old message: {err}"); +} + +/// done_when 3, unknown id: refused as a gap in apr's table, with somewhere to report it. +#[test] +fn test_3601_unknown_id_is_refused_as_a_gap_in_apr() { + for id in [trueno_quant::GGML_TYPE_COUNT, 4242] { + let reader = one_tensor(&[64], id, &payload(64), 0); + let err = reader + .get_tensor_raw("t.weight") + .expect_err("not a ggml type at the pin") + .to_string(); + for needle in [ + &id.to_string(), + "newer than the ggml type table", + "not a corrupt file", + "github.com/paiml/aprender/issues", + ] { + assert!(err.contains(needle), "id {id}: missing {needle:?} in: {err}"); + } + // `try_gguf_raw_import` (GH-375) falls back to the dequant path on these two + // phrases; a sizing refusal must not trip it. + assert!(!err.contains("cannot represent exactly") && !err.contains("not yet supported")); + } +} + +/// #3656: sized is not readable. `get_tensor_f32` used to return `(b - 128) * 0.01` per +/// raw byte for ids 16..=23 and `Ok`; every IQ/TQ tensor must now be refused by type. +/// With #3601 sizing these ids, raw import reaches the GH-375 fallback — so this refusal +/// is what stops that fallback from importing invented weights. +#[test] +fn test_3656_get_tensor_f32_refuses_iq_and_tq_instead_of_approximating() { + for (id, name, blck, type_size) in IQ_AND_TQ { + let reader = one_tensor(&[blck as u64], id, &payload(type_size), 0); + let err = reader + .get_tensor_f32("t.weight") + .expect_err("no dequantizer here: must refuse, never approximate") + .to_string(); + for needle in ["t.weight", name, &format!("ggml type {id}"), "no dequantizer"] { + assert!(err.contains(needle), "{name}: missing {needle:?} in: {err}"); + } + assert!( + !err.contains("cannot represent exactly") && !err.contains("not yet supported"), + "{name}: a dequant refusal must not read as a GH-375 fallback trigger: {err}" + ); + } +} + +/// done_when 1 at the library boundary `apr inspect` calls: RosettaStone inspects an +/// IQ4_XS GGUF, names the dtype, and reports its true on-disk size. +#[test] +fn test_3601_rosetta_inspect_accepts_an_iq4_xs_gguf() { + use std::io::Write; + + let bytes = build_synthetic_gguf_with_tensor("blk.0.ffn_up.weight", &[512], 23, &payload(272), &[]); + let mut file = tempfile::NamedTempFile::with_suffix(".gguf").expect("temp file"); + file.write_all(&bytes).expect("write GGUF"); + file.flush().expect("flush"); + + let report = crate::format::rosetta::RosettaStone::new() + .inspect(file.path()) + .expect("an IQ4_XS GGUF must inspect"); + let t = report + .tensors + .iter() + .find(|t| t.name == "blk.0.ffn_up.weight") + .expect("tensor listed"); + assert_eq!(t.dtype, "IQ4_XS"); + assert_eq!(t.size_bytes, 2 * 136); +} + +/// `apr tensors` checks every tensor's extent against the file before listing. With +/// IQ2_XXS weighted at 144/256 bytes/element it declared a correctly sized file +/// "Truncated" — measured on Qwen3.5-0.8B-UD-IQ2_XXS.gguf. Sized right, it lists it. +#[test] +fn test_3601_tensor_listing_accepts_a_correctly_sized_iq2_xxs_gguf() { + let bytes = build_synthetic_gguf_with_tensor("t.weight", &[512], 16, &payload(2 * 66), &[]); + let listing = crate::format::tensors::list_tensors_from_bytes( + &bytes, + crate::format::tensors::TensorListOptions::default(), + ) + .expect("a correctly sized IQ2_XXS GGUF must list"); + assert_eq!(listing.tensors.len(), 1); + assert_eq!(listing.tensors[0].dtype, "IQ2_XXS"); + assert_eq!(listing.tensors[0].size_bytes, 2 * 66); +} diff --git a/crates/aprender-core/src/format/gguf/shape.rs b/crates/aprender-core/src/format/gguf/shape.rs index c224f1704c..2643808f14 100644 --- a/crates/aprender-core/src/format/gguf/shape.rs +++ b/crates/aprender-core/src/format/gguf/shape.rs @@ -128,20 +128,20 @@ impl GgufReader { // Q6_K - dequantize (super blocks of 256 elements, 210 bytes/block) dequantize_q6_k(&self.data, tensor_start, num_elements)? } - // I-quants (importance matrix quantization) - complex formats - // For now, approximate with simpler dequantization - 16..=23 => { - // IQ2_XXS=16, IQ2_XS=17, IQ2_S=18, IQ3_XXS=19, IQ3_S=20, IQ1_S=21, IQ4_NL=22, IQ4_XS=23 - // Fall back to zero-filled tensor with warning - eprintln!( - "Warning: I-quant dtype {} for tensor '{}' not fully supported, using approximation", - meta.dtype, name - ); - dequantize_iq_approximate(&self.data, tensor_start, num_elements, meta.dtype) - } + // #3656: IQ types (16..=23) used to go to `dequantize_iq_approximate`, which + // mapped each raw byte to `(b - 128) * 0.01` and returned Ok — `apr convert` + // wrote those invented weights (std 36.6x the real tensor) and exited 0. With + // no real dequantizer here, the only honest answer is a refusal. _ => { + let type_name = trueno_quant::GgmlType::from_id(meta.dtype) + .map_or("a type ggml does not define", trueno_quant::GgmlType::as_str); return Err(AprenderError::FormatError { - message: format!("Unsupported GGUF dtype {} for tensor '{name}'", meta.dtype), + message: format!( + "GGUF tensor '{name}' is {type_name} (ggml type {}): aprender-core has \ + no dequantizer for it, so it cannot be read as F32. Refusing rather \ + than approximating, which would invent its weights", + meta.dtype + ), }); } }; @@ -174,9 +174,9 @@ impl GgufReader { /// Get raw tensor bytes without dequantization (preserves Q4K/Q6K) /// - /// Returns (raw_bytes, shape, ggml_dtype) where dtype is per GGML spec: - /// - 0=F32, 1=F16, 2=Q4_0, 3=Q4_1, 8=Q8_0 - /// - 10=Q2_K, 11=Q3_K, 12=Q4_K, 13=Q5_K, 14=Q6_K + /// Returns (raw_bytes, shape, ggml_dtype) where dtype is the GGML type id. Every + /// type live in upstream ggml is sized (`trueno_quant::TRAITS`); an id upstream + /// removed, or one newer than that table, is refused by name. pub fn get_tensor_raw(&self, name: &str) -> Result<(Vec, Vec, u32)> { let meta = self .tensors @@ -211,37 +211,20 @@ impl GgufReader { let tensor_start = self.data_offset + meta.offset as usize; - // Calculate byte size based on dtype (GGML dtype values) - // See llama.cpp ggml.h for type definitions - // GGML enum: 0=F32, 1=F16, 2=Q4_0, 3=Q4_1, 6=Q5_0, 7=Q5_1, 8=Q8_0, 9=Q8_1 - // 10=Q2_K, 11=Q3_K, 12=Q4_K, 13=Q5_K, 14=Q6_K, 15=Q8_K + // #3601: size from ggml's own `type_traits` (`trueno_quant::TRAITS`, extracted + // from upstream and fixture-checked by PMAT-3430) rather than a hand-typed match. + // That match knew 15 of ggml's 35 live ids, so every IQ*/TQ* tensor was refused + // as "Unsupported dtype 23 for raw extraction" — which read as a corrupt file. + // Sizing a type is not a claim that anything here can dequantize it. + let ggml_type = trueno_quant::GgmlType::try_from_id(meta.dtype) + .map_err(|e| unsized_ggml_type_error(name, &e))?; + + // Whole blocks, rounding DOWN: the arithmetic the hand-typed arms used, kept so + // no id that was already sized changes its byte count. // BUG-GGUF-002 FIX: Use checked arithmetic to prevent overflow in byte size calc - // Note: Some dtypes share byte sizes but are documented separately for clarity - #[allow(clippy::match_same_arms)] - let byte_size = match meta.dtype { - 0 => num_elements.checked_mul(4), // F32 - 1 => num_elements.checked_mul(2), // F16 - 2 => (num_elements / 32).checked_mul(18), // Q4_0: 32 elements = 2 (d) + 16 (qs) - 3 => (num_elements / 32).checked_mul(20), // Q4_1: 32 elements = 2 (d) + 2 (m) + 16 (qs) - // dtype 4,5 = removed (Q4_2, Q4_3) - 6 => (num_elements / 32).checked_mul(22), // Q5_0: 32 elements = 2 (d) + 4 (qh) + 16 (ql) - 7 => (num_elements / 32).checked_mul(24), // Q5_1: 32 elements = 2 (d) + 2 (m) + 4 (qh) + 16 (ql) - 8 => (num_elements / 32).checked_mul(34), // Q8_0: 32 elements = 2 (d) + 32 (qs) - 9 => (num_elements / 32).checked_mul(36), // Q8_1: 32 elements = 2 (d) + 2 (s) + 32 (qs) - 10 => (num_elements / 256).checked_mul(84), // Q2_K: 256 elements = 84 bytes - 11 => (num_elements / 256).checked_mul(110), // Q3_K: 256 elements = 110 bytes - 12 => (num_elements / 256).checked_mul(144), // Q4_K: 256 elements = 144 bytes - 13 => (num_elements / 256).checked_mul(176), // Q5_K: 256 elements = 176 bytes - 14 => (num_elements / 256).checked_mul(210), // Q6_K: 256 elements = 210 bytes - 15 => (num_elements / 256).checked_mul(292), // Q8_K: 256 elements = 292 bytes - 30 => num_elements.checked_mul(2), // BF16: 2 bytes per element - _ => { - return Err(AprenderError::FormatError { - message: format!("Unsupported dtype {} for raw extraction", meta.dtype), - }); - } - } - .ok_or_else(|| AprenderError::FormatError { + let byte_size = (num_elements / ggml_type.block_size()) + .checked_mul(ggml_type.block_bytes()) + .ok_or_else(|| AprenderError::FormatError { message: format!( "Tensor '{}' byte size calculation overflow (dtype: {})", name, meta.dtype @@ -270,3 +253,28 @@ impl GgufReader { Ok(result) } } + +/// #3601: the refusal for a tensor whose ggml type id has no size in `trueno_quant::TRAITS`. +/// +/// Every id live upstream is sized, so this is reached only by an id upstream REMOVED or one +/// newer than the pinned table. The message says which — "Unsupported dtype 23" sent readers +/// looking for file corruption when the gap was in apr. +fn unsized_ggml_type_error(tensor: &str, e: &trueno_quant::GgmlTypeError) -> AprenderError { + let why = match e { + trueno_quant::GgmlTypeError::Removed { .. } => { + "no current ggml defines a layout for it, so the file must be re-quantized from \ + its source weights" + } + trueno_quant::GgmlTypeError::Unknown { .. } => { + "it is newer than the ggml type table this apr was built with (pinned by \ + scripts/llama_pin.toml) — a gap in apr, not a corrupt file; please report it at \ + https://github.com/paiml/aprender/issues" + } + }; + AprenderError::FormatError { + message: format!( + "GGUF tensor '{tensor}': {e}; {why}. The ids apr sizes are listed in \ + contracts/ggml-type-v1.yaml" + ), + } +} diff --git a/crates/aprender-core/src/format/model_arithmetic.rs b/crates/aprender-core/src/format/model_arithmetic.rs index f49f54719c..3c97ae1675 100644 --- a/crates/aprender-core/src/format/model_arithmetic.rs +++ b/crates/aprender-core/src/format/model_arithmetic.rs @@ -43,7 +43,9 @@ //! does not settle, so it is left unbound and `QE2E-BND-005` stays false. use crate::format::layout_contract::block_sizes; -use crate::format::model_family::{MlpType, ModelConstraints, ModelSizeConfig}; +use crate::format::model_family::{ + AttentionType, DeltaNetShape, MlpType, ModelConstraints, ModelSizeConfig, +}; // ============================================================================ // Equation: model_parameter_count @@ -96,13 +98,16 @@ pub struct ParameterBreakdown { /// `d_attn`, `d_ffn`, `d_norm` for one ordinary (softmax-attention) decoder /// layer of a family described by `size` + `constraints`. /// -/// - `d_attn` = `d*(n_h*d_k) + 2*d*(n_kv*d_k) + (n_h*d_k)*d` (Q, K, V, O), -/// plus the four bias vectors when `constraints.has_bias`. +/// - `d_attn` = `d*q_out + 2*d*(n_kv*d_k) + (n_h*d_k)*d` (Q, K, V, O), plus the +/// four bias vectors when `constraints.has_bias`, plus `2*d_k` of q/k norm +/// weights when `constraints.qk_norm`. `q_out` is `n_h*d_k`, or twice that +/// for a gated-attention family (see the comment on the `q_out` binding). /// - `d_ffn` = `3*d*d_ff` for a gated MLP (SwiGLU/GeGLU), else `2*d*d_ff`. /// - `d_norm` = `2*d` (input norm + post-attention norm). /// -/// This is the dense/GQA accounting. It is NOT the Gated DeltaNet accounting: -/// see [`model_parameter_count`] for what that needs. +/// This is the softmax-attention layer. It is NOT the Gated DeltaNet +/// accounting: that is [`gated_deltanet_layer_params`], and +/// [`hybrid_layers`] interleaves the two. #[must_use] pub fn attention_layer_params( size: &ModelSizeConfig, @@ -113,32 +118,144 @@ pub fn attention_layer_params( let q_dim = (size.num_heads as u64).saturating_mul(d_k); let kv_dim = (size.num_kv_heads as u64).saturating_mul(d_k); + // A gated-attention family emits the output gate from the q projection, so + // that matrix is 2*n_h*d_k wide rather than n_h*d_k. MEASURED in + // Qwen3.5-0.8B-Q4_K_M.gguf: attn_q is [1024, 4096] while attn_output is + // [2048, 1024], so o_proj still sees n_h*d_k = 2048 and only q is doubled. + let q_out = if matches!( + constraints.attention_type, + AttentionType::HybridGatedDeltaNet + ) { + q_dim.saturating_mul(2) + } else { + q_dim + }; let projections = d - .saturating_mul(q_dim) + .saturating_mul(q_out) .saturating_add(d.saturating_mul(kv_dim).saturating_mul(2)) .saturating_add(q_dim.saturating_mul(d)); + // A bias vector is as wide as the projection it biases, so the q bias is + // q_out — doubled with the matrix for a gated family. No shipped family + // exercises this today (Qwen3.5 has no attention bias; Qwen2.5 has biases + // but is not gated, so q_out == q_dim), which is exactly how the original + // `q_dim` here survived a delta-0 measurement: found by the AD-04 quorum + // on #3350, not by any model. let biases = if constraints.has_bias { - q_dim + q_out .saturating_add(kv_dim.saturating_mul(2)) .saturating_add(d) } else { 0 }; + // Per-head q/k RMSNorm weights (attn_q_norm/attn_k_norm, one d_k vector each). + let qk_norms = if constraints.qk_norm { + d_k.saturating_mul(2) + } else { + 0 + }; - let d_ff = size.intermediate_dim as u64; + LayerParams { + d_attn: projections.saturating_add(biases).saturating_add(qk_norms), + d_ffn: ffn_params(size, constraints), + d_norm: d.saturating_mul(2), + } +} + +/// `d_ffn` for one layer: `3*d*d_ff` for a gated MLP (SwiGLU/GeGLU — gate, up +/// and down), else `2*d*d_ff`. Both layer kinds of a hybrid model share it. +fn ffn_params(size: &ModelSizeConfig, constraints: &ModelConstraints) -> u64 { let matrices = if matches!(constraints.mlp_type, MlpType::SwiGlu | MlpType::GatedMlp) { 3 } else { 2 }; + (size.hidden_dim as u64) + .saturating_mul(size.intermediate_dim as u64) + .saturating_mul(matrices) +} + +/// `d_attn`, `d_ffn`, `d_norm` for one **Gated DeltaNet** layer — the `d_attn` +/// the dense formula cannot express, because none of its dimensions are +/// `n_h * d_k`. +/// +/// Every term is one tensor of `Qwen3.5-0.8B-Q4_K_M.gguf`, named here as the +/// file names it (`i` = `inner_size`, `s` = `state_size`, `k` = `conv_kernel`, +/// `h` = `group_count`): +/// +/// | Tensor | Shape | Parameters | +/// |--------|-------|------------| +/// | `attn_qkv.weight` | `[d, 3*i]` | `3*d*i` | +/// | `attn_gate.weight` | `[d, i]` | `d*i` | +/// | `ssm_conv1d.weight` | `[k, 3*i]` | `3*k*i` | +/// | `ssm_alpha.weight`, `ssm_beta.weight` | `[d, h]` each | `2*d*h` | +/// | `ssm_a`, `ssm_dt.bias` | `[h]` each | `2*h` | +/// | `ssm_norm.weight` | `[s]` | `s` | +/// | `ssm_out.weight` | `[i, d]` | `i*d` | +/// +/// `d_norm` is `2*d` (`attn_norm` + `post_attention_norm`) and `d_ffn` is the +/// same SwiGLU block as an attention layer — a DeltaNet layer differs only in +/// how it mixes tokens. +#[must_use] +pub fn gated_deltanet_layer_params( + size: &ModelSizeConfig, + constraints: &ModelConstraints, + shape: &DeltaNetShape, +) -> LayerParams { + let d = size.hidden_dim as u64; + let inner = shape.inner_size as u64; + let heads = shape.group_count as u64; + let qkv = d.saturating_mul(inner).saturating_mul(3); + let gate = d.saturating_mul(inner); + let conv = (shape.conv_kernel as u64) + .saturating_mul(inner) + .saturating_mul(3); + let alpha_beta = d.saturating_mul(heads).saturating_mul(2); + let per_head = heads.saturating_mul(2); + let out = inner.saturating_mul(d); + + let d_attn = qkv + .saturating_add(gate) + .saturating_add(conv) + .saturating_add(alpha_beta) + .saturating_add(per_head) + .saturating_add(shape.state_size as u64) + .saturating_add(out); LayerParams { - d_attn: projections.saturating_add(biases), - d_ffn: d.saturating_mul(d_ff).saturating_mul(matrices), + d_attn, + d_ffn: ffn_params(size, constraints), d_norm: d.saturating_mul(2), } } +/// The per-layer input for a HYBRID model: Gated DeltaNet layers with a +/// softmax-attention layer every `full_attention_interval`-th position, the +/// last of each group. +/// +/// Falls back to [`uniform_layers`] for any family that declares no DeltaNet +/// shape (every family but `qwen3_5`) or declares no schedule, so the answer +/// for a dense family is byte-for-byte what it was before #3346. +#[must_use] +pub fn hybrid_layers(size: &ModelSizeConfig, constraints: &ModelConstraints) -> Vec { + let Some(shape) = constraints.deltanet else { + return uniform_layers(size, constraints); + }; + if shape.full_attention_interval == 0 { + return uniform_layers(size, constraints); + } + let attention = attention_layer_params(size, constraints); + let deltanet = gated_deltanet_layer_params(size, constraints, &shape); + (0..size.num_layers) + .map(|i| { + if (i + 1) % shape.full_attention_interval == 0 { + attention + } else { + deltanet + } + }) + .collect() +} + /// `L` copies of [`attention_layer_params`] — the per-layer input for a /// homogeneous (non-hybrid) model of `size.num_layers` layers. #[must_use] @@ -166,14 +283,19 @@ pub fn uniform_layers(size: &ModelSizeConfig, constraints: &ModelConstraints) -> /// /// # What this does NOT discharge /// -/// `QE2E-INV-001` wants `P(Qwen3.5-9B) ∈ [9.0B, 9.2B]`. Feeding this function -/// [`uniform_layers`] for the 9B variant yields ≈8.21B, because Qwen3.5 is -/// `hybrid_gated_deltanet`: three of every four layers are Gated DeltaNet, whose -/// `d_attn` covers conv, gate and state projections sized by `inner_size`, -/// `state_size`, `conv_kernel` and `group_count`. Those four keys exist in -/// `contracts/model-families/qwen3_5.yaml` but NOT in [`ModelConstraints`], so -/// the GDN `d_attn` cannot be derived from the config type as it stands. The -/// equation is implemented; the 9B invariant is not verified. +/// `QE2E-INV-001` wants `P(Qwen3.5-9B) ∈ [9.0B, 9.2B]`, and it is still NOT +/// discharged — but for a different reason than before #3346. +/// +/// The arithmetic is now verified against a real file: fed the configuration of +/// `Qwen3.5-0.8B-Q4_K_M.gguf`, [`hybrid_layers`] + this function reproduce that +/// file's 320-tensor inventory EXACTLY (752,393,024 parameters). The Gated +/// DeltaNet shape reaches it through [`ModelConstraints::deltanet`]. +/// +/// Applying the same, now-falsified, arithmetic to the 9b variant of +/// `contracts/model-families/qwen3_5.yaml` gives **8,344,907,136** — 0.655B +/// below the range. The remaining gap is in the DESCRIPTOR, not here, and is +/// not something this function may paper over: see +/// `model_arithmetic_tests.rs::qwen35_9b_hybrid_layers_still_fall_short_of_the_invariant_range`. #[must_use] pub fn model_parameter_count( size: &ModelSizeConfig, diff --git a/crates/aprender-core/src/format/model_arithmetic_tests.rs b/crates/aprender-core/src/format/model_arithmetic_tests.rs index f80d810730..941925e46b 100644 --- a/crates/aprender-core/src/format/model_arithmetic_tests.rs +++ b/crates/aprender-core/src/format/model_arithmetic_tests.rs @@ -4,7 +4,9 @@ //! the formula changes) and, where the contract states one, one property. use super::*; -use crate::format::model_family::{Activation, AttentionType, NormType, PositionalEncoding}; +use crate::format::model_family::{ + Activation, AttentionType, DeltaNetShape, NormType, PositionalEncoding, +}; /// A four-dimension toy model whose parameter count is small enough to verify /// by hand: V=10, d=4, L=2, n_h=2, n_kv=1, d_k=2, d_ff=8. @@ -52,9 +54,233 @@ fn qwen35_constraints() -> ModelConstraints { positional_encoding: PositionalEncoding::Rope, mlp_type: MlpType::SwiGlu, qk_norm: true, + deltanet: Some(DeltaNetShape { + inner_size: 2048, + state_size: 128, + conv_kernel: 4, + group_count: 8, + full_attention_interval: 4, + }), } } +// --------------------------------------------------------------------------- +// Ground truth: a REAL Qwen3.5 file (#3346) +// --------------------------------------------------------------------------- +// +// Every number below was read out of `~/models/Qwen3.5-0.8B-Q4_K_M.gguf` +// (sha256 `bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517`, +// GGUF v3, 320 tensors) by parsing the file header directly on 2026-09-16 — +// not from a model card, a memory, or the family descriptor. The descriptor +// `contracts/model-families/qwen3_5.yaml` declares only the 9b and 27b +// variants, and no Qwen3.5-9B file is on this box, so the 0.8B file is the +// only Qwen3.5 whose true tensor inventory can be MEASURED here. It is the +// oracle for the shape arithmetic: if the config-derived count and this +// inventory disagree, the arithmetic is wrong. +// +// Shapes are GGUF `ne` order (`[in, out]` for a 2-D weight); only the element +// COUNT matters for a parameter total, so the order is reproduced verbatim +// rather than transposed. + +/// One Gated DeltaNet layer of `Qwen3.5-0.8B-Q4_K_M.gguf` (measured: `blk.0`, +/// identical for the 18 layers whose index is not `interval-1 mod interval`). +const QWEN35_0_8B_GDN_LAYER: &[(&str, &[usize])] = &[ + ("attn_gate.weight", &[1024, 2048]), + ("attn_norm.weight", &[1024]), + ("attn_qkv.weight", &[1024, 6144]), + ("ffn_down.weight", &[3584, 1024]), + ("ffn_gate.weight", &[1024, 3584]), + ("ffn_up.weight", &[1024, 3584]), + ("post_attention_norm.weight", &[1024]), + ("ssm_a", &[16]), + ("ssm_alpha.weight", &[1024, 16]), + ("ssm_beta.weight", &[1024, 16]), + ("ssm_conv1d.weight", &[4, 6144]), + ("ssm_dt.bias", &[16]), + ("ssm_norm.weight", &[128]), + ("ssm_out.weight", &[2048, 1024]), +]; + +/// One full-attention layer of the same file (measured: `blk.3`, identical for +/// the 6 layers at indices 3, 7, 11, 15, 19, 23 — `full_attention_interval` 4). +/// +/// Two shapes here are NOT what dense/GQA accounting predicts, and both are +/// facts of the file: `attn_q` is `[1024, 4096]` = `2 * num_heads * head_dim` +/// (Qwen3.5 gates the attention output, so the q projection emits the gate +/// alongside the query — `attn_output` is `[2048, 1024]`, confirming +/// `num_heads * head_dim` = 2048), and `attn_q_norm`/`attn_k_norm` are present +/// at `head_dim`. +const QWEN35_0_8B_ATTENTION_LAYER: &[(&str, &[usize])] = &[ + ("attn_k.weight", &[1024, 512]), + ("attn_k_norm.weight", &[256]), + ("attn_norm.weight", &[1024]), + ("attn_output.weight", &[2048, 1024]), + ("attn_q.weight", &[1024, 4096]), + ("attn_q_norm.weight", &[256]), + ("attn_v.weight", &[1024, 512]), + ("ffn_down.weight", &[3584, 1024]), + ("ffn_gate.weight", &[1024, 3584]), + ("ffn_up.weight", &[1024, 3584]), + ("post_attention_norm.weight", &[1024]), +]; + +/// The file's non-layer tensors. There is no `output.weight`: the 0.8B TIES +/// its unembedding to the embedding, unlike the 9b descriptor. +const QWEN35_0_8B_GLOBAL: &[(&str, &[usize])] = &[ + ("output_norm.weight", &[1024]), + ("token_embd.weight", &[1024, 248_320]), +]; + +/// Number of GDN and full-attention layers in the measured file (24 blocks, +/// `qwen35.full_attention_interval` = 4). +const QWEN35_0_8B_GDN_LAYERS: u64 = 18; +const QWEN35_0_8B_ATTENTION_LAYERS: u64 = 6; + +/// Total elements of a measured tensor list. +fn tensor_elements(tensors: &[(&str, &[usize])]) -> u64 { + tensors + .iter() + .map(|(_, dims)| u64::try_from(dims.iter().product::()).unwrap_or(u64::MAX)) + .sum() +} + +/// `Qwen3.5-0.8B` as the GGUF's own metadata keys describe it: `block_count` +/// 24, `embedding_length` 1024, `feed_forward_length` 3584, +/// `attention.head_count` 8, `attention.head_count_kv` 2, `attention.key_length` +/// 256, `rope.freq_base` 1e7, `attention.layer_norm_rms_epsilon` 1e-6, and a +/// 248320-token vocabulary (`token_embd.weight` is `[1024, 248320]`). +fn qwen35_0_8b_size() -> ModelSizeConfig { + ModelSizeConfig { + parameters: "0.8B".to_string(), + hidden_dim: 1024, + num_layers: 24, + num_heads: 8, + num_kv_heads: 2, + intermediate_dim: 3584, + vocab_size: 248_320, + max_position_embeddings: 262_144, + head_dim: 256, + rope_theta: 10_000_000.0, + norm_eps: 1e-6, + } +} + +/// The measured file total: `752,393,024` parameters. +const QWEN35_0_8B_MEASURED_TOTAL: u64 = 752_393_024; + +#[test] +fn qwen35_0_8b_measured_inventory_sums_to_the_file_total() { + // Tensor COUNT: 14 per GDN layer, 11 per attention layer, 2 global = 320, + // which is what the GGUF header declares (`n_tensors`). + let counted = QWEN35_0_8B_GDN_LAYER.len() * 18 + QWEN35_0_8B_ATTENTION_LAYER.len() * 6 + 2; + assert_eq!(counted, 320, "GGUF header declares 320 tensors"); + + assert_eq!(tensor_elements(QWEN35_0_8B_GDN_LAYER), 21_555_360); + assert_eq!(tensor_elements(QWEN35_0_8B_ATTENTION_LAYER), 18_352_640); + assert_eq!(tensor_elements(QWEN35_0_8B_GLOBAL), 254_280_704); + + let total = tensor_elements(QWEN35_0_8B_GLOBAL) + + QWEN35_0_8B_GDN_LAYERS * tensor_elements(QWEN35_0_8B_GDN_LAYER) + + QWEN35_0_8B_ATTENTION_LAYERS * tensor_elements(QWEN35_0_8B_ATTENTION_LAYER); + assert_eq!(total, QWEN35_0_8B_MEASURED_TOTAL); +} + +/// The defect of #3346, as a number rather than a claim: one layer kind applied +/// to all 24 layers cannot reproduce a REAL hybrid file. +/// +/// Before #3346 this shortfall was 107,992,896 (14.4%) against the then-dense +/// `attention_layer_params`. The dense baseline is gone — that function now +/// models the gated q projection and the q/k norms the file actually has — so +/// what is left to measure is the mixer itself: 18 of the 24 layers are Gated +/// DeltaNet, and at these dimensions a DeltaNet layer is LARGER than an +/// attention layer (21,555,360 against 18,352,640), so uniform accounting is +/// short by 57,648,960. +/// +/// The sign is not universal, which is the point: at the 9b descriptor's +/// dimensions the same comparison inverts (see +/// `qwen35_9b_uniform_layers_are_the_wrong_model_for_a_hybrid_family`), because +/// that descriptor keeps `inner_size: 2048` while quadrupling `hidden_dim`. +/// Only the real schedule gets a hybrid model right. +#[test] +fn uniform_accounting_cannot_reproduce_the_measured_qwen35_0_8b_file() { + let size = qwen35_0_8b_size(); + let constraints = qwen35_0_8b_constraints(); + let p = model_parameter_count(&size, &constraints, &uniform_layers(&size, &constraints)); + + assert_eq!(p.total, 694_744_064); + assert_eq!(QWEN35_0_8B_MEASURED_TOTAL - p.total, 57_648_960); + assert!( + tensor_elements(QWEN35_0_8B_GDN_LAYER) > tensor_elements(QWEN35_0_8B_ATTENTION_LAYER), + "at 0.8B dims the DeltaNet layer is the bigger of the two" + ); +} + +/// The same constraints, but as the MEASURED 0.8B file declares them: its +/// `ssm.group_count` is 16 (not the 9b descriptor's 8), it ties its +/// unembedding, and its full-attention layers carry q/k norms. +fn qwen35_0_8b_constraints() -> ModelConstraints { + ModelConstraints { + tied_embeddings: true, + deltanet: Some(DeltaNetShape { + inner_size: 2048, + state_size: 128, + conv_kernel: 4, + group_count: 16, + full_attention_interval: 4, + }), + ..qwen35_constraints() + } +} + +/// #3346 acceptance. Fed the 0.8B configuration, the equation must reproduce +/// the measured file EXACTLY — not approximately, and not after tuning a +/// constant. Both layer kinds are checked separately so a failure names which +/// block is mis-shaped rather than only that the total drifted. +#[test] +fn qwen35_0_8b_config_derived_count_equals_the_measured_gguf_inventory() { + let size = qwen35_0_8b_size(); + let constraints = qwen35_0_8b_constraints(); + let shape = constraints + .deltanet + .expect("the 0.8B constraints declare a DeltaNet shape"); + + assert_eq!( + gated_deltanet_layer_params(&size, &constraints, &shape).total(), + tensor_elements(QWEN35_0_8B_GDN_LAYER), + "Gated DeltaNet layer" + ); + assert_eq!( + attention_layer_params(&size, &constraints).total(), + tensor_elements(QWEN35_0_8B_ATTENTION_LAYER), + "full-attention layer" + ); + + let layers = hybrid_layers(&size, &constraints); + assert_eq!(layers.len(), 24); + let p = model_parameter_count(&size, &constraints, &layers); + assert_eq!(p.total, QWEN35_0_8B_MEASURED_TOTAL); +} + +/// The hybrid schedule is measured, not assumed: in the real file the layers +/// carrying `attn_q`/`attn_k`/`attn_v` are exactly 3, 7, 11, 15, 19, 23 — every +/// `full_attention_interval`-th layer, counting the LAST of each group. +#[test] +fn the_hybrid_schedule_puts_full_attention_last_in_each_group() { + let size = qwen35_0_8b_size(); + let constraints = qwen35_0_8b_constraints(); + let shape = constraints.deltanet.expect("declared"); + let attn = attention_layer_params(&size, &constraints); + let full: Vec = hybrid_layers(&size, &constraints) + .iter() + .enumerate() + .filter(|(_, l)| **l == attn) + .map(|(i, _)| i) + .collect(); + + assert_eq!(full, vec![3, 7, 11, 15, 19, 23]); + assert_eq!(shape.full_attention_interval, 4); +} + // --------------------------------------------------------------------------- // model_parameter_count // --------------------------------------------------------------------------- @@ -62,13 +288,17 @@ fn qwen35_constraints() -> ModelConstraints { #[test] fn worked_example_attention_layer_params() { let p = attention_layer_params(&toy_size(), &qwen35_constraints()); - // d_attn = d*q_dim + 2*d*kv_dim + q_dim*d = 4*4 + 2*4*2 + 4*4 = 48 - assert_eq!(p.d_attn, 48); + // Qwen3.5 gates the attention output, so q_out = 2*q_dim (MEASURED: + // attn_q [1024, 4096] against attn_output [2048, 1024] in the 0.8B file), + // and qk_norm adds one d_k vector each for attn_q_norm/attn_k_norm. + // d_attn = d*q_out + 2*d*kv_dim + q_dim*d + 2*d_k + // = 4*8 + 2*4*2 + 4*4 + 2*2 = 68 + assert_eq!(p.d_attn, 68); // d_ffn = 3*d*d_ff = 3*4*8 = 96 assert_eq!(p.d_ffn, 96); // d_norm = 2*d = 8 assert_eq!(p.d_norm, 8); - assert_eq!(p.total(), 152); + assert_eq!(p.total(), 172); } #[test] @@ -79,10 +309,10 @@ fn worked_example_model_parameter_count() { let p = model_parameter_count(&size, &constraints, &layers); assert_eq!(p.embedding, 40); // V*d = 10*4 - assert_eq!(p.layers, 304); // L*(48+96+8) = 2*152 + assert_eq!(p.layers, 344); // L*(68+96+8) = 2*172 assert_eq!(p.final_norm, 4); // d_final = d assert_eq!(p.unembedding, 40); // untied lm_head = V*d - assert_eq!(p.total, 388); // P = 40 + 304 + 4 + 40 + assert_eq!(p.total, 428); // P = 40 + 344 + 4 + 40 } #[test] @@ -94,7 +324,7 @@ fn tied_embeddings_drop_the_trailing_v_times_d() { let p = model_parameter_count(&size, &constraints, &layers); assert_eq!(p.unembedding, 0); - assert_eq!(p.total, 348); // 388 - 40 + assert_eq!(p.total, 388); // 428 - 40 } #[test] @@ -103,30 +333,90 @@ fn bias_adds_exactly_the_four_projection_bias_vectors() { let mut constraints = qwen35_constraints(); constraints.has_bias = true; let p = attention_layer_params(&size, &constraints); - // q_dim + 2*kv_dim + d = 4 + 4 + 4 = 12 - assert_eq!(p.d_attn, 48 + 12); + // qwen35_constraints() is a GATED family, so the q projection is q_out = + // 2*q_dim wide and its bias vector is too: q_out + 2*kv_dim + d + // = 8 + 4 + 4 = 16. This test asserted 12 (a q_dim-wide bias under a + // q_out-wide matrix) until the quorum on #3350 read the two lines against + // each other; a bias narrower than its projection is not a model. + assert_eq!(p.d_attn, 68 + 16); +} + +#[test] +fn a_non_gated_family_with_biases_still_counts_a_q_dim_wide_q_bias() { + // The other polarity: where q_out == q_dim (every non-gated family), the + // fix above must change nothing. + let size = toy_size(); + let mut constraints = qwen35_constraints(); + constraints.attention_type = AttentionType::Gqa; + constraints.has_bias = true; + let p = attention_layer_params(&size, &constraints); + let without = { + let mut c = constraints.clone(); + c.has_bias = false; + attention_layer_params(&size, &c) + }; + // q_dim + 2*kv_dim + d = 4 + 4 + 4 = 12, on top of the un-doubled matrix. + assert_eq!(p.d_attn - without.d_attn, 12); } -/// QE2E-INV-001 wants `P(Qwen3.5-9B) ∈ [9.0B, 9.2B]`. Dense/GQA accounting -/// gives 8.21B, because three of every four Qwen3.5 layers are Gated DeltaNet -/// and their `d_attn` is sized by `inner_size`/`state_size`/`conv_kernel`/ -/// `group_count`, none of which exist in `ModelConstraints`. This test pins the -/// number so the gap is a measured fact, not a claim. +/// The premise this test used to carry — "the four DeltaNet keys do not exist +/// in `ModelConstraints`" — stopped being true in #3346, so it states what is +/// true now: [`uniform_layers`] is the WRONG model for a hybrid family in +/// either direction. Pretending all 32 layers run softmax attention OVER-counts +/// the mixer (8.745B) exactly as pretending they are all dense under-counted it +/// before; only [`hybrid_layers`] describes the architecture. #[test] -fn qwen35_9b_uniform_layers_do_not_reach_the_invariant_range() { +fn qwen35_9b_uniform_layers_are_the_wrong_model_for_a_hybrid_family() { let size = qwen35_9b_size(); let constraints = qwen35_constraints(); - let layers = uniform_layers(&size, &constraints); + let uniform = model_parameter_count(&size, &constraints, &uniform_layers(&size, &constraints)); + let hybrid = model_parameter_count(&size, &constraints, &hybrid_layers(&size, &constraints)); + + assert_eq!(uniform.embedding, 1_017_118_720); + assert_eq!(uniform.unembedding, 1_017_118_720); + assert_eq!(uniform.total, 8_745_406_464); + assert!( + uniform.total > hybrid.total, + "a full-attention layer is bigger than a DeltaNet layer at 9B dims" + ); +} + +/// QE2E-INV-001 wants `P(Qwen3.5-9B) ∈ [9.0B, 9.2B]`. With the DeltaNet shape +/// carried and the arithmetic falsified against a real file, the 9b descriptor +/// yields **8,344,907,136** — still 0.655B short. The obligation therefore +/// stays UNPROVED, and this test exists to keep it that way: the honest move is +/// to pin the number, not to widen the range until it passes. +/// +/// What is unaccounted for is in the descriptor, and it is visible in the +/// numbers it declares. `inner_size: 2048` is the value the 0.8B file uses at +/// `hidden_dim` 1024, i.e. `2*d`; at the 9b's `hidden_dim` 4096 the same 2048 +/// makes the mixer NARROWER than the residual stream it mixes, and it is also +/// inconsistent with the 9b's own `group_count: 8` (8 * 128 != 2048, whereas +/// the measured 0.8B satisfies 16 * 128 == 2048). Either the 9b `inner_size` +/// and `group_count` were copied from the small variant, or the range was +/// copied from a model card. Only a real Qwen3.5-9B file can tell them apart, +/// and no such file is on this box — see #3346. +#[test] +fn qwen35_9b_hybrid_layers_still_fall_short_of_the_invariant_range() { + let size = qwen35_9b_size(); + let constraints = qwen35_constraints(); + let layers = hybrid_layers(&size, &constraints); let p = model_parameter_count(&size, &constraints, &layers); - assert_eq!(p.embedding, 1_017_118_720); - assert_eq!(p.unembedding, 1_017_118_720); - assert_eq!(p.total, 8_208_519_168); + assert_eq!(layers.len(), 32); + assert_eq!(p.total, 8_344_907_136); assert!( p.total < 9_000_000_000, - "QE2E-INV-001 is NOT discharged by dense accounting: got {}", + "QE2E-INV-001 is still NOT discharged: got {}", p.total ); + + // The descriptor's own self-inconsistency, as a fact rather than a remark. + let shape = constraints.deltanet.expect("9b declares a DeltaNet shape"); + assert!( + !shape.heads_span_the_mixer(), + "9b declares group_count * state_size != inner_size" + ); } /// A hybrid model is a slice of DIFFERENT per-layer costs — the reason the diff --git a/crates/aprender-core/src/format/model_family.rs b/crates/aprender-core/src/format/model_family.rs index ccea26f952..f35192896c 100644 --- a/crates/aprender-core/src/format/model_family.rs +++ b/crates/aprender-core/src/format/model_family.rs @@ -247,6 +247,50 @@ pub struct ModelSizeConfig { pub norm_eps: f64, } +/// #3346: the Gated DeltaNet shape of a hybrid family. +/// +/// `contracts/model-families/qwen3_5.yaml` declares `inner_size`, +/// `state_size`, `conv_kernel`, `group_count` and `full_attention_interval` +/// under `constraints:`, and until #3346 [`ModelConstraints`] carried none of +/// them. A DeltaNet layer's parameters live entirely in these dimensions — the +/// conv, the in/out projections and the gates — so a type that drops them +/// cannot count a hybrid model: dense accounting under-counted the real +/// `Qwen3.5-0.8B-Q4_K_M.gguf` by 14.4%. +/// +/// Tensor names below are the GGUF names of that measured file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DeltaNetShape { + /// Width of the DeltaNet mixer — `attn_gate`/`ssm_out` are `inner_size` + /// wide and `attn_qkv`/`ssm_conv1d` are `3 * inner_size` wide. + pub inner_size: usize, + /// Per-head state width; `ssm_norm` is a vector of this length. + pub state_size: usize, + /// Depthwise conv width: `ssm_conv1d` is `[conv_kernel, 3 * inner_size]`. + pub conv_kernel: usize, + /// Number of DeltaNet heads — the length of `ssm_a` and `ssm_dt.bias`, and + /// the output width of `ssm_alpha`/`ssm_beta`. + pub group_count: usize, + /// Period of the hybrid schedule: every `full_attention_interval`-th layer + /// runs softmax attention instead, the LAST of each group (measured: layers + /// 3, 7, 11, 15, 19, 23 of 24 at interval 4). + pub full_attention_interval: usize, +} + +impl DeltaNetShape { + /// `group_count * state_size`, which must equal `inner_size` for the + /// declared shape to describe one mixer. + /// + /// This is a CHECK, not a repair: the measured 0.8B satisfies it + /// (16 * 128 = 2048) and the 9b descriptor does not (8 * 128 != 2048). + #[must_use] + pub const fn heads_span_the_mixer(&self) -> bool { + match self.group_count.checked_mul(self.state_size) { + Some(span) => span == self.inner_size, + None => false, + } + } +} + /// Architectural constraints for a model family. #[derive(Debug, Clone)] pub struct ModelConstraints { @@ -259,6 +303,9 @@ pub struct ModelConstraints { pub mlp_type: MlpType, /// GH-280: Whether Q and K projections have per-head RMSNorm (e.g., Qwen3) pub qk_norm: bool, + /// #3346: Gated DeltaNet shape, for hybrid families that declare one. + /// `None` for every family whose descriptor has no `inner_size`. + pub deltanet: Option, } /// Tensor name template for a model family. diff --git a/crates/aprender-core/src/format/model_family_contract_falsify.rs b/crates/aprender-core/src/format/model_family_contract_falsify.rs index d801f98337..a9bb70b243 100644 --- a/crates/aprender-core/src/format/model_family_contract_falsify.rs +++ b/crates/aprender-core/src/format/model_family_contract_falsify.rs @@ -869,6 +869,56 @@ mod contract_falsification { ); } + // ======================================================================== + // FALSIFY-MF-QWEN35-010: the Gated DeltaNet shape keys survive the loader + // + // Prediction: loading contracts/model-families/qwen3_5.yaml yields + // constraints.deltanet = Some(inner 2048, state 128, conv 4, + // group 8, interval 4) — the values the descriptor declares — + // and EVERY other family yields None. + // If fails: the keys are declared in YAML and dropped on the way into + // ModelConstraints, which is the #3346 defect. That drop made the + // arithmetic under-count a real Qwen3.5 file by 14.4%, because a + // DeltaNet layer's parameters live entirely in these dimensions. + // ======================================================================== + #[test] + fn falsify_mf_qwen35_010_deltanet_shape_reaches_constraints() { + let families = load_all_families(); + let qwen35 = families + .iter() + .find(|(name, _)| name == "qwen3_5") + .expect("FALSIFIED: qwen3_5 family not found"); + + let shape = qwen35 + .1 + .constraints + .deltanet + .expect("FALSIFIED QWEN35-010: qwen3_5 declares inner_size/state_size, got None"); + + assert_eq!( + shape, + DeltaNetShape { + inner_size: 2048, + state_size: 128, + conv_kernel: 4, + group_count: 8, + full_attention_interval: 4, + }, + "FALSIFIED QWEN35-010: loader did not reproduce the declared shape" + ); + + // No other family declares a DeltaNet mixer, so none may acquire one: + // a false Some() here would change that family's parameter accounting. + for (name, config) in &families { + if name != "qwen3_5" { + assert!( + config.constraints.deltanet.is_none(), + "FALSIFIED QWEN35-010: {name} acquired a DeltaNet shape it never declared" + ); + } + } + } + // ======================================================================== // FALSIFY-MF-QWEN35-007: Qwen3.5 architecture class registered // diff --git a/crates/aprender-core/src/format/model_family_loader.rs b/crates/aprender-core/src/format/model_family_loader.rs index 76e1dc996b..0d792337fe 100644 --- a/crates/aprender-core/src/format/model_family_loader.rs +++ b/crates/aprender-core/src/format/model_family_loader.rs @@ -13,8 +13,8 @@ use std::path::Path; use crate::error::{AprenderError, Result}; use crate::format::model_family::{ - Activation, AttentionType, CertificationConfig, ChatTemplateConfig, DynModelFamily, - FamilyRegistry, GgufFusionRule, GgufTensorTemplate, MlpType, ModelConstraints, + Activation, AttentionType, CertificationConfig, ChatTemplateConfig, DeltaNetShape, + DynModelFamily, FamilyRegistry, GgufFusionRule, GgufTensorTemplate, MlpType, ModelConstraints, ModelFamilyConfig, ModelSizeConfig, NormType, PositionalEncoding, ShapeTemplate, TensorTemplate, }; diff --git a/crates/aprender-core/src/format/model_family_tests.rs b/crates/aprender-core/src/format/model_family_tests.rs index 327719fefc..95992f411f 100644 --- a/crates/aprender-core/src/format/model_family_tests.rs +++ b/crates/aprender-core/src/format/model_family_tests.rs @@ -122,6 +122,7 @@ mod tests { positional_encoding: PositionalEncoding::Rope, mlp_type: MlpType::SwiGlu, qk_norm: false, + deltanet: None, }, tensor_template: TensorTemplate { embedding: "model.embed_tokens.weight".to_string(), diff --git a/crates/aprender-core/src/format/parsing.rs b/crates/aprender-core/src/format/parsing.rs index 6ae3911bef..cdfca2f99a 100644 --- a/crates/aprender-core/src/format/parsing.rs +++ b/crates/aprender-core/src/format/parsing.rs @@ -73,6 +73,24 @@ fn parse_constraints(yaml: &YamlValue) -> Result { )?, mlp_type: MlpType::from_str_contract(yaml.get_str("mlp_type").unwrap_or("gelu_mlp"))?, qk_norm: yaml.get_bool("qk_norm").unwrap_or(false), + deltanet: parse_deltanet_shape(yaml), + }) +} + +/// #3346: the Gated DeltaNet shape keys of a `constraints:` block. +/// +/// `inner_size` and `state_size` are what make the block a DeltaNet mixer, so +/// both are required; a descriptor declaring neither (every family but +/// `qwen3_5`) yields `None` and the dense accounting it had before. +fn parse_deltanet_shape(yaml: &YamlValue) -> Option { + let inner_size = yaml.get_usize("inner_size")?; + let state_size = yaml.get_usize("state_size")?; + Some(DeltaNetShape { + inner_size, + state_size, + conv_kernel: yaml.get_usize("conv_kernel").unwrap_or(0), + group_count: yaml.get_usize("group_count").unwrap_or(0), + full_attention_interval: yaml.get_usize("full_attention_interval").unwrap_or(0), }) } diff --git a/crates/aprender-core/src/format/safetensors.rs b/crates/aprender-core/src/format/safetensors.rs index 7aa564f355..ff6337fe5d 100644 --- a/crates/aprender-core/src/format/safetensors.rs +++ b/crates/aprender-core/src/format/safetensors.rs @@ -7,16 +7,13 @@ /// Unknown dtypes default to 4.0 (F32 size) as a conservative overestimate. /// See contracts/gguf-kquant-element-size-v1.yaml (PMAT-869). fn ggml_dtype_element_size(dtype: u32) -> f64 { - // Index: [F32, F16, Q4_0, Q4_1, (4), (5), Q5_0, Q5_1, Q8_0, Q8_1, - // Q2_K, Q3_K, Q4_K, Q5_K, Q6_K, Q8_K, IQ2_XXS, IQ2_XS, - // IQ3_XXS, IQ1_S, IQ4_NL, IQ3_S, IQ2_S, IQ4_XS, I8, I16, - // BF16, I32, I64, F64, IQ1_M] - const SIZES: [f64; 31] = [ - 4.0, 2.0, 0.5625, 0.625, 4.0, 4.0, 0.6875, 0.75, 1.0625, 1.125, 0.328_125, 0.429_687_5, - 0.5625, 0.6875, 0.820_312_5, 1.140_625, 0.5625, 0.625, 0.6875, 0.4375, 0.5625, 0.4375, - 0.625, 0.5, 1.0, 2.0, 2.0, 4.0, 8.0, 8.0, 0.375, - ]; - SIZES.get(dtype as usize).copied().unwrap_or(4.0) + // #3601: type_size / blck_size from ggml's own type table (`trueno_quant::TRAITS`, + // PMAT-3430). The hand-typed SIZES copy this replaces had every IQ quant but IQ4_NL + // wrong (IQ2_XXS 144/256 for 66/256 — so `apr tensors` refused a valid UD-IQ2_XXS + // GGUF as "Truncated") and its 26-30 tail misordered (BF16 at 0.375 bytes/element). + trueno_quant::GgmlType::from_id(dtype).map_or(4.0, |t| { + t.block_bytes() as f64 / t.block_size() as f64 + }) } /// List tensors from GGUF file bytes diff --git a/crates/aprender-core/src/format/tensors.rs b/crates/aprender-core/src/format/tensors.rs index e0117c896f..e48a21007f 100644 --- a/crates/aprender-core/src/format/tensors.rs +++ b/crates/aprender-core/src/format/tensors.rs @@ -569,16 +569,11 @@ fn list_tensors_v1(data: &[u8], options: TensorListOptions) -> Result &'static str { - // Indexed by GGML type code (ggml.h GGML_TYPE_*). The tail (24-30) was - // misordered — BF16 was at 26, mislabeling BF16 tensors (code 30) as - // "IQ1_M", I32 (26) as "BF16", etc. Correct order per ggml.h: - // I8=24, I16=25, I32=26, I64=27, F64=28, IQ1_M=29, BF16=30. - const NAMES: [&str; 31] = [ - "F32", "F16", "Q4_0", "Q4_1", "unknown", "unknown", "Q5_0", "Q5_1", "Q8_0", "Q8_1", "Q2_K", - "Q3_K", "Q4_K", "Q5_K", "Q6_K", "Q8_K", "IQ2_XXS", "IQ2_XS", "IQ3_XXS", "IQ1_S", "IQ4_NL", - "IQ3_S", "IQ2_S", "IQ4_XS", "I8", "I16", "I32", "I64", "F64", "IQ1_M", "BF16", - ]; - NAMES.get(dtype as usize).copied().unwrap_or("unknown") + // #3601: read from ggml's own type table (`trueno_quant::TRAITS`, PMAT-3430) instead + // of a hand-typed copy. The copy stopped at id 30, so TQ1_0/TQ2_0 (34/35) — which + // `get_tensor_raw` now sizes — would have printed "unknown"; an earlier copy had the + // 24-30 tail misordered. Removed and never-defined ids stay "unknown". + trueno_quant::GgmlType::from_id(dtype).map_or("unknown", trueno_quant::GgmlType::as_str) } include!("safetensors.rs"); diff --git a/crates/aprender-core/src/format/tensors_tests_stats.rs b/crates/aprender-core/src/format/tensors_tests_stats.rs index f65106c3ea..5fbbc7a26c 100644 --- a/crates/aprender-core/src/format/tensors_tests_stats.rs +++ b/crates/aprender-core/src/format/tensors_tests_stats.rs @@ -317,24 +317,26 @@ fn test_ggml_dtype_element_size_exhaustive() { assert!((super::ggml_dtype_element_size(14) - 0.820_312_5).abs() < 0.01); // Q8_K = 292/256 (PMAT-869: was 1.0625) assert!((super::ggml_dtype_element_size(15) - 1.140_625).abs() < 0.01); - // BF16 - assert!((super::ggml_dtype_element_size(26) - 2.0).abs() < 0.001); - // I-quant types - assert!((super::ggml_dtype_element_size(16) - 0.5625).abs() < 0.01); // IQ2_XXS - assert!((super::ggml_dtype_element_size(17) - 0.625).abs() < 0.01); // IQ2_XS - assert!((super::ggml_dtype_element_size(18) - 0.6875).abs() < 0.01); // IQ3_XXS - assert!((super::ggml_dtype_element_size(19) - 0.4375).abs() < 0.01); // IQ1_S - assert!((super::ggml_dtype_element_size(20) - 0.5625).abs() < 0.01); // IQ4_NL - assert!((super::ggml_dtype_element_size(21) - 0.4375).abs() < 0.01); // IQ3_S - assert!((super::ggml_dtype_element_size(22) - 0.625).abs() < 0.01); // IQ2_S - assert!((super::ggml_dtype_element_size(23) - 0.5).abs() < 0.01); // IQ4_XS - // Integer types - assert!((super::ggml_dtype_element_size(24) - 1.0).abs() < 0.01); // I8 - assert!((super::ggml_dtype_element_size(25) - 2.0).abs() < 0.01); // I16 - assert!((super::ggml_dtype_element_size(27) - 4.0).abs() < 0.01); // I32 - assert!((super::ggml_dtype_element_size(28) - 8.0).abs() < 0.01); // I64 - assert!((super::ggml_dtype_element_size(29) - 8.0).abs() < 0.01); // F64 - assert!((super::ggml_dtype_element_size(30) - 0.375).abs() < 0.01); // IQ1_M + // #3601: the rows below used to pin the hand-typed table's values under its own + // labels — the 24-30 tail shifted by one (26 "BF16", 30 "IQ1_M") and seven IQ rows + // wrong. ggml.h: I8=24, I16=25, I32=26, I64=27, F64=28, IQ1_M=29, BF16=30. + // I-quant types (block_bytes / block_elems, ggml-common.h) + assert!((super::ggml_dtype_element_size(16) - 66.0 / 256.0).abs() < 1e-9); // IQ2_XXS + assert!((super::ggml_dtype_element_size(17) - 74.0 / 256.0).abs() < 1e-9); // IQ2_XS + assert!((super::ggml_dtype_element_size(18) - 98.0 / 256.0).abs() < 1e-9); // IQ3_XXS + assert!((super::ggml_dtype_element_size(19) - 50.0 / 256.0).abs() < 1e-9); // IQ1_S + assert!((super::ggml_dtype_element_size(20) - 18.0 / 32.0).abs() < 1e-9); // IQ4_NL + assert!((super::ggml_dtype_element_size(21) - 110.0 / 256.0).abs() < 1e-9); // IQ3_S + assert!((super::ggml_dtype_element_size(22) - 82.0 / 256.0).abs() < 1e-9); // IQ2_S + assert!((super::ggml_dtype_element_size(23) - 136.0 / 256.0).abs() < 1e-9); // IQ4_XS + // Integer, F64, IQ1_M and BF16 — in ggml's order + assert!((super::ggml_dtype_element_size(24) - 1.0).abs() < 1e-9); // I8 + assert!((super::ggml_dtype_element_size(25) - 2.0).abs() < 1e-9); // I16 + assert!((super::ggml_dtype_element_size(26) - 4.0).abs() < 1e-9); // I32 + assert!((super::ggml_dtype_element_size(27) - 8.0).abs() < 1e-9); // I64 + assert!((super::ggml_dtype_element_size(28) - 8.0).abs() < 1e-9); // F64 + assert!((super::ggml_dtype_element_size(29) - 56.0 / 256.0).abs() < 1e-9); // IQ1_M + assert!((super::ggml_dtype_element_size(30) - 2.0).abs() < 1e-9); // BF16 // Unknown defaults to F32 (4.0) — conservative size estimate assert!((super::ggml_dtype_element_size(99) - 4.0).abs() < 0.001); assert!((super::ggml_dtype_element_size(u32::MAX) - 4.0).abs() < 0.001); @@ -385,6 +387,46 @@ fn test_kquant_element_size_exact_block_ratio_pmat869() { assert_eq!(q2k_bytes, 84, "256 Q2_K elements must report 84 bytes"); } +// ==================================================================== +// #3601: every other hand-typed row. bytes/element = block_bytes / block_elems from +// ggml-common.h. The SIZES table this replaces had seven IQ rows wrong and the 26-30 +// tail shifted by one (BF16 read IQ1_M's 0.375). +// ==================================================================== + +#[test] +fn test_3601_element_size_iq_tq_and_the_24_to_30_tail() { + // (dtype code, block_bytes, block_elems, label) + let cases: [(u32, f64, f64, &str); 18] = [ + (16, 66.0, 256.0, "IQ2_XXS"), // was 0.5625: `apr tensors` called a valid file truncated + (17, 74.0, 256.0, "IQ2_XS"), + (18, 98.0, 256.0, "IQ3_XXS"), + (19, 50.0, 256.0, "IQ1_S"), + (20, 18.0, 32.0, "IQ4_NL"), + (21, 110.0, 256.0, "IQ3_S"), + (22, 82.0, 256.0, "IQ2_S"), + (23, 136.0, 256.0, "IQ4_XS"), + (24, 1.0, 1.0, "I8"), + (25, 2.0, 1.0, "I16"), + (26, 4.0, 1.0, "I32"), // was 2.0 + (27, 8.0, 1.0, "I64"), // was 4.0 + (28, 8.0, 1.0, "F64"), + (29, 56.0, 256.0, "IQ1_M"), // was 8.0 + (30, 2.0, 1.0, "BF16"), // was 0.375 + (34, 54.0, 256.0, "TQ1_0"), // was the 4.0 default + (35, 66.0, 256.0, "TQ2_0"), // was the 4.0 default + (8, 34.0, 32.0, "Q8_0"), // an unchanged row, for contrast + ]; + for (code, block_bytes, block_elems, label) in cases { + let expected = block_bytes / block_elems; + let actual = super::ggml_dtype_element_size(code); + assert_eq!(actual, expected, "{label} (code {code})"); + } + // Removed upstream and never-defined ids keep the conservative F32 default. + for code in [4, 5, 31, 36, 43, 4242] { + assert_eq!(super::ggml_dtype_element_size(code), 4.0, "code {code}"); + } +} + // ==================================================================== // Coverage: f16_to_f32 additional edge cases // ==================================================================== diff --git a/crates/aprender-core/src/format/tensors_tests_workflow.rs b/crates/aprender-core/src/format/tensors_tests_workflow.rs index a72740fdb5..ec6f2467b7 100644 --- a/crates/aprender-core/src/format/tensors_tests_workflow.rs +++ b/crates/aprender-core/src/format/tensors_tests_workflow.rs @@ -50,6 +50,29 @@ fn test_ggml_dtype_name_unknown() { assert_eq!(ggml_dtype_name(255), "unknown"); } +/// #3601: names come from `trueno_quant::TRAITS` now. Ids 0-30 render exactly as the +/// hand-typed table did (transcribed below), and the ids past it are named rather than +/// "unknown" — `get_tensor_raw` sizes TQ1_0/TQ2_0, so inspect must be able to say them. +#[test] +fn test_3601_ggml_dtype_name_keeps_0_to_30_and_names_the_ids_past_it() { + const PRE_3601: [&str; 31] = [ + "F32", "F16", "Q4_0", "Q4_1", "unknown", "unknown", "Q5_0", "Q5_1", "Q8_0", "Q8_1", "Q2_K", + "Q3_K", "Q4_K", "Q5_K", "Q6_K", "Q8_K", "IQ2_XXS", "IQ2_XS", "IQ3_XXS", "IQ1_S", "IQ4_NL", + "IQ3_S", "IQ2_S", "IQ4_XS", "I8", "I16", "I32", "I64", "F64", "IQ1_M", "BF16", + ]; + for (id, name) in PRE_3601.iter().enumerate() { + assert_eq!(ggml_dtype_name(id as u32), *name, "id {id}"); + } + let named = [(34, "TQ1_0"), (35, "TQ2_0"), (39, "MXFP4"), (40, "NVFP4"), (41, "Q1_0"), (42, "Q2_0")]; + for (id, name) in named { + assert_eq!(ggml_dtype_name(id), name, "id {id}"); + } + // Removed upstream (repacked at load time) and past the table: still "unknown". + for id in [31, 32, 33, 36, 37, 38, 43] { + assert_eq!(ggml_dtype_name(id), "unknown", "id {id}"); + } +} + #[test] fn test_ggml_dtype_element_size() { assert!((ggml_dtype_element_size(0) - 4.0).abs() < 0.001); // F32 diff --git a/crates/aprender-serve/Cargo.toml b/crates/aprender-serve/Cargo.toml index 3b142207b9..dabf141a8b 100644 --- a/crates/aprender-serve/Cargo.toml +++ b/crates/aprender-serve/Cargo.toml @@ -103,6 +103,8 @@ reqwest = { version = "0.12", features = ["json", "blocking"], optional = true } # Serialization (for REST API, not ML code) serde = { version = "1", features = ["derive"] } serde_json = "1" +# #3604: the F2 guard receipt is keyed on the model file's sha256. +sha2 = { workspace = true } # Lock-free concurrent data structures arc-swap = { version = "1.7", optional = true } diff --git a/crates/aprender-serve/src/api/model_source.rs b/crates/aprender-serve/src/api/model_source.rs index 1b93e38c5d..fee58973e3 100644 --- a/crates/aprender-serve/src/api/model_source.rs +++ b/crates/aprender-serve/src/api/model_source.rs @@ -201,27 +201,12 @@ fn read_magic(path: &Path) -> Option> { /// /// Derived from the qtype actually stored in the loaded tensors, which is the /// ground truth — `general.file_type` in GGUF metadata is advisory and is -/// known to go stale when a file is requantized. +/// known to go stale when a file is requantized. The names come from the ggml type +/// table; the hand-typed copy this replaced knew 15 ids and reported no quantization +/// at all for every IQ type (#3662). #[must_use] pub fn gguf_qtype_name(qtype: u32) -> Option<&'static str> { - Some(match qtype { - 0 => "F32", - 1 => "F16", - 2 => "Q4_0", - 3 => "Q4_1", - 6 => "Q5_0", - 7 => "Q5_1", - 8 => "Q8_0", - 9 => "Q8_1", - 10 => "Q2_K", - 11 => "Q3_K", - 12 => "Q4_K", - 13 => "Q5_K", - 14 => "Q6_K", - 15 => "Q8_K", - 30 => "BF16", - _ => return None, - }) + trueno_quant::GgmlType::from_id(qtype).map(trueno_quant::GgmlType::as_str) } #[cfg(test)] @@ -313,4 +298,17 @@ mod tests { // Unknown ids must not be reported as some plausible quantization. assert_eq!(gguf_qtype_name(9999), None); } + + /// #3662: the hand-typed table knew 15 ids, so every IQ quantization reported + /// no quantization at all. + #[test] + fn qtype_names_cover_every_live_family() { + assert_eq!(gguf_qtype_name(10), Some("Q2_K")); + assert_eq!(gguf_qtype_name(16), Some("IQ2_XXS")); + assert_eq!(gguf_qtype_name(23), Some("IQ4_XS")); + assert_eq!(gguf_qtype_name(30), Some("BF16")); + // Removed upstream (Q4_2, Q4_3): not a quantization anything can hold. + assert_eq!(gguf_qtype_name(4), None); + assert_eq!(gguf_qtype_name(5), None); + } } diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs new file mode 100644 index 0000000000..788892cd6c --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -0,0 +1,332 @@ +//! The F2 hybrid guard's receipt: validate once per (model, apr version, +//! device), and let later runs read the answer instead of re-deriving it. +//! +//! # Why (#3604, operator ruling 2026-09-20) +//! +//! `f2_validate_qwen35` proves the CUDA hybrid forward against a CPU reference +//! before it will serve a token. Measured on lambda 4090 with Qwen3.5-4B-Q4_K_M +//! (#3598 row 1): that guard is **67 % of a 14 s time-to-first-token**, and +//! 90–93 % of the guard is the CPU reference forward. It re-derives the same +//! answer for the same three inputs on every `apr run`. +//! +//! The guard is right to exist and wrong to run per call. So: the first run of +//! a (model sha256, apr version, device) triple validates and writes a receipt; +//! a later run whose triple matches reads the receipt and skips the forward; +//! `--revalidate` forces a fresh run and rewrites it. +//! +//! # The receipt IS the validation — which is why it is strict +//! +//! Every path that is not "a receipt whose three keys all match" validates: +//! +//! - no receipt → validate. Absence is never consent (`done_when` 4). +//! - unreadable or malformed receipt → validate, and say why. +//! - any one of the three keys differs → validate, naming the key +//! (`done_when` 3 — the three planted-receipt falsifiers live in +//! `f2_receipt_tests.rs`). +//! - `--revalidate` → validate, even on a perfect match (`done_when` 2). +//! +//! And a receipt is only ever WRITTEN after a validation that actually judged +//! something. The guard has three early-`true` exits that judge nothing — +//! `SKIP_PARITY_GATE=1`, a probe shorter than two tokens, a CPU reference that +//! failed to run — and none of them may launder itself into a receipt, or a +//! one-token prompt would "validate" the triple for every prompt after it. +//! +//! # What this module is not +//! +//! It knows nothing about CUDA and compiles without the `cuda` feature, so its +//! decision table is unit-tested on every build. The GPU-facing wrapper that +//! calls it lives beside `f2_validate_qwen35` in `forward_qwen35.rs`. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// The three things a validation is a statement about. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct F2ReceiptKey { + /// sha256 of the model file's bytes, lower-case hex. The whole file: a + /// prefix or a size+mtime fingerprint would let a planted receipt with the + /// wrong hash pass, which is the first falsifier. + pub model_sha256: String, + /// The version of the crate that ran the guard. + pub apr_version: String, + /// The device the GPU half ran on, as the driver names it. + pub device: String, +} + +/// What a passed validation leaves behind. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct F2Receipt { + /// Format version of this file, so a future shape change re-validates + /// instead of misreading. + pub schema: u32, + /// The triple this receipt vouches for. + #[serde(flatten)] + pub key: F2ReceiptKey, + /// Unix seconds when the validation passed. + pub validated_at: u64, + /// How many probe positions the passing validation actually compared. + pub positions_judged: usize, +} + +/// The current receipt schema. Bump it and every old receipt re-validates. +pub const F2_RECEIPT_SCHEMA: u32 = 1; + +/// Why a run is validating instead of reading the receipt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum F2ValidateReason { + /// No receipt exists for this model. + NoReceipt, + /// A file exists but could not be read or parsed. + Unreadable(String), + /// The receipt is for a different schema version. + SchemaMismatch { + /// What the file says. + found: u32, + /// What this build writes. + expected: u32, + }, + /// The receipt is for a different model. + ModelSha256Mismatch { + /// The hash in the file. + found: String, + /// The hash of the model being loaded. + expected: String, + }, + /// The receipt was written by a different apr version. + AprVersionMismatch { + /// The version in the file. + found: String, + /// This crate's version. + expected: String, + }, + /// The receipt was written for a different device. + DeviceMismatch { + /// The device in the file. + found: String, + /// The device this run is on. + expected: String, + }, + /// The user asked for a fresh run. + Revalidate, +} + +impl std::fmt::Display for F2ValidateReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoReceipt => write!(f, "no receipt for this model"), + Self::Unreadable(e) => write!(f, "receipt unreadable ({e})"), + Self::SchemaMismatch { found, expected } => { + write!(f, "receipt schema {found}, this build writes {expected}") + }, + Self::ModelSha256Mismatch { found, expected } => write!( + f, + "receipt is for model {}…, this file is {}…", + &found[..found.len().min(12)], + &expected[..expected.len().min(12)] + ), + Self::AprVersionMismatch { found, expected } => { + write!(f, "receipt written by apr {found}, this is {expected}") + }, + Self::DeviceMismatch { found, expected } => { + write!(f, "receipt written for {found}, this device is {expected}") + }, + Self::Revalidate => write!(f, "--revalidate"), + } + } +} + +/// The decision, and its evidence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum F2Decision { + /// Read the receipt; skip the forward. + Skip { + /// The receipt that matched. + receipt: F2Receipt, + }, + /// Run the forward. + Validate(F2ValidateReason), +} + +/// THE DECISION TABLE. Pure: no filesystem, no clock, no env, so the three +/// planted-receipt falsifiers and the absence case are ordinary unit tests. +/// +/// `found` is what the reader returned: `Ok(None)` for no file, `Err` for a +/// file that would not read or parse, `Ok(Some)` for a parsed receipt. +#[must_use] +pub fn decide( + found: Result, String>, + expected: &F2ReceiptKey, + revalidate: bool, +) -> F2Decision { + // The user's word comes first: a perfect receipt does not survive + // --revalidate, and the reason names the flag rather than the receipt. + if revalidate { + return F2Decision::Validate(F2ValidateReason::Revalidate); + } + let receipt = match found { + Err(e) => return F2Decision::Validate(F2ValidateReason::Unreadable(e)), + Ok(None) => return F2Decision::Validate(F2ValidateReason::NoReceipt), + Ok(Some(r)) => r, + }; + if receipt.schema != F2_RECEIPT_SCHEMA { + return F2Decision::Validate(F2ValidateReason::SchemaMismatch { + found: receipt.schema, + expected: F2_RECEIPT_SCHEMA, + }); + } + // Each key is compared and NAMED on its own. A combined "keys differ" would + // hide which of the three moved, and the three falsifiers are one per key. + if receipt.key.model_sha256 != expected.model_sha256 { + return F2Decision::Validate(F2ValidateReason::ModelSha256Mismatch { + found: receipt.key.model_sha256, + expected: expected.model_sha256.clone(), + }); + } + if receipt.key.apr_version != expected.apr_version { + return F2Decision::Validate(F2ValidateReason::AprVersionMismatch { + found: receipt.key.apr_version, + expected: expected.apr_version.clone(), + }); + } + if receipt.key.device != expected.device { + return F2Decision::Validate(F2ValidateReason::DeviceMismatch { + found: receipt.key.device, + expected: expected.device.clone(), + }); + } + F2Decision::Skip { receipt } +} + +/// Where receipts live. `APR_F2_RECEIPT_DIR` wins (tests and operators pin it), +/// then `$XDG_CACHE_HOME/apr/f2-receipts`, then `$HOME/.cache/apr/f2-receipts`. +/// `None` when no home can be found — the caller then validates every time and +/// says so; it never errors. +#[must_use] +pub fn receipt_dir() -> Option { + if let Some(d) = std::env::var_os("APR_F2_RECEIPT_DIR") { + return Some(PathBuf::from(d)); + } + if let Some(x) = std::env::var_os("XDG_CACHE_HOME") { + if !x.is_empty() { + return Some(PathBuf::from(x).join("apr").join("f2-receipts")); + } + } + std::env::var_os("HOME").map(|h| { + PathBuf::from(h) + .join(".cache") + .join("apr") + .join("f2-receipts") + }) +} + +/// One file per model. The apr version and device are INSIDE the file and +/// compared by [`decide`], so a device swap on the same box shows up as a +/// named mismatch rather than a second silent file. +#[must_use] +pub fn receipt_path(dir: &Path, model_sha256: &str) -> PathBuf { + dir.join(format!("{model_sha256}.json")) +} + +/// Read a receipt. `Ok(None)` when the file does not exist; `Err` for anything +/// else, because "it was there and I could not read it" must not look like +/// "it was not there" — both validate, but the operator should see which. +pub fn read_receipt(path: &Path) -> Result, String> { + let text = match std::fs::read_to_string(path) { + Ok(t) => t, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(format!("{}: {e}", path.display())), + }; + serde_json::from_str::(&text) + .map(Some) + .map_err(|e| format!("{}: not a receipt: {e}", path.display())) +} + +/// Write a receipt atomically: a temp file PRIVATE TO THIS WRITER, then a +/// rename. A crash mid-write leaves the old receipt or none, and two `apr run` +/// processes validating the same model at once each rename their own complete +/// file — the last rename wins whole, never half of the other's. +/// +/// The first version of this used one shared `.json.tmp`. The AD-04 +/// quorum on #3634 (lane 1) read that against the docstring's "never a +/// truncated one" and was right: with a shared name, writer B truncates the +/// file writer A is about to rename, and A renames B's partial into place. The +/// reader would classify it `Unreadable` and validate — the safe direction — +/// but the atomicity claim was false. The temp name now carries the pid and a +/// per-process counter, so no two writers share one. +pub fn write_receipt(path: &Path, receipt: &F2Receipt) -> Result<(), String> { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + + let dir = path + .parent() + .ok_or_else(|| format!("{}: no parent directory", path.display()))?; + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let stem = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("receipt"); + let tmp = dir.join(format!( + ".{stem}.{}.{}.tmp", + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + let body = serde_json::to_string_pretty(receipt).map_err(|e| e.to_string())?; + if let Err(e) = std::fs::write(&tmp, body) { + return Err(format!("{}: {e}", tmp.display())); + } + // If the rename fails, do not leave the private temp behind to be mistaken + // for anything: it carries no meaning once it is not the receipt. + std::fs::rename(&tmp, path).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + format!("{}: {e}", path.display()) + }) +} + +/// sha256 of the model bytes, lower-case hex. +/// +/// The WHOLE file, deliberately. On a 2.5 GB Q4_K_M this is measurable (the +/// receipt for this ticket reports it), but it is the only identity under +/// which the first falsifier holds: a planted receipt with the wrong hash must +/// re-validate, and a cheaper identity that the planted receipt could still +/// satisfy would pass it. +#[must_use] +pub fn model_sha256(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(bytes); + let out = h.finalize(); + let mut s = String::with_capacity(64); + for b in out { + use std::fmt::Write as _; + let _ = write!(s, "{b:02x}"); + } + s +} + +/// The version of this crate — the one that ran the guard. +#[must_use] +pub fn apr_version() -> String { + env!("CARGO_PKG_VERSION").to_string() +} + +/// Now, in unix seconds; 0 if the clock is before the epoch. +#[must_use] +pub fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) +} + +/// Did the user ask for a fresh run? `apr run --revalidate` sets this before +/// inference starts — the same env seam the guard already uses for +/// `SKIP_PARITY_GATE`, chosen over threading a bool through six signatures that +/// another open PR (#3606) is changing at the same time. +#[must_use] +pub fn revalidate_requested() -> bool { + std::env::var("APR_F2_REVALIDATE").is_ok_and(|v| v == "1") +} + +#[cfg(test)] +#[path = "f2_receipt_tests.rs"] +mod f2_receipt_tests; diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs new file mode 100644 index 0000000000..448adc179b --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs @@ -0,0 +1,260 @@ +//! #3604 `done_when` 2, 3 and 4, as tests that can fail. +//! +//! The decision table is pure, so every case here is a plain call — no GPU, +//! no model file, no clock. The three PLANTED-RECEIPT falsifiers (`done_when` +//! 3) are the point of this file: each plants a receipt that is perfect in two +//! keys and wrong in the third, and asserts the decision is `Validate` naming +//! THAT key. A cache that skipped on any of them would be serving a GPU path +//! that was proved for a different model, a different build, or a different +//! device. + +use super::{ + apr_version, decide, model_sha256, read_receipt, receipt_path, write_receipt, F2Decision, + F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, +}; + +fn key() -> F2ReceiptKey { + F2ReceiptKey { + model_sha256: "a".repeat(64), + apr_version: "0.68.2".to_string(), + device: "NVIDIA GeForce RTX 4090".to_string(), + } +} + +fn receipt_for(key: &F2ReceiptKey) -> F2Receipt { + F2Receipt { + schema: F2_RECEIPT_SCHEMA, + key: key.clone(), + validated_at: 1_758_400_000, + positions_judged: 64, + } +} + +// ---------------------------------------------------------------- done_when 1 +#[test] +fn a_receipt_whose_three_keys_all_match_is_read_and_the_forward_is_skipped() { + let k = key(); + let r = receipt_for(&k); + assert_eq!( + decide(Ok(Some(r.clone())), &k, false), + F2Decision::Skip { receipt: r } + ); +} + +// ---------------------------------------------------------------- done_when 3 +// THE THREE PLANTED-RECEIPT FALSIFIERS. Two keys right, one wrong, every time. + +#[test] +fn falsifier_a_planted_receipt_with_the_wrong_model_sha256_revalidates() { + let k = key(); + let mut planted = receipt_for(&k); + planted.key.model_sha256 = "b".repeat(64); // right apr, right device, wrong model + match decide(Ok(Some(planted)), &k, false) { + F2Decision::Validate(F2ValidateReason::ModelSha256Mismatch { found, expected }) => { + assert_eq!(found, "b".repeat(64)); + assert_eq!(expected, "a".repeat(64)); + }, + other => panic!("a wrong-model receipt must re-validate naming the model, got {other:?}"), + } +} + +#[test] +fn falsifier_a_planted_receipt_from_a_different_apr_version_revalidates() { + let k = key(); + let mut planted = receipt_for(&k); + planted.key.apr_version = "0.61.0".to_string(); // right model, right device, older build + match decide(Ok(Some(planted)), &k, false) { + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { found, expected }) => { + assert_eq!(found, "0.61.0"); + assert_eq!(expected, "0.68.2"); + }, + other => { + panic!("a wrong-version receipt must re-validate naming the version, got {other:?}") + }, + } +} + +#[test] +fn falsifier_a_planted_receipt_for_a_different_device_revalidates() { + let k = key(); + let mut planted = receipt_for(&k); + planted.key.device = "NVIDIA GB10".to_string(); // right model, right apr, other GPU + match decide(Ok(Some(planted)), &k, false) { + F2Decision::Validate(F2ValidateReason::DeviceMismatch { found, expected }) => { + assert_eq!(found, "NVIDIA GB10"); + assert_eq!(expected, "NVIDIA GeForce RTX 4090"); + }, + other => panic!("a wrong-device receipt must re-validate naming the device, got {other:?}"), + } +} + +// ---------------------------------------------------------------- done_when 4 +#[test] +fn a_missing_receipt_validates_because_absence_is_never_consent() { + assert_eq!( + decide(Ok(None), &key(), false), + F2Decision::Validate(F2ValidateReason::NoReceipt) + ); +} + +#[test] +fn an_unreadable_receipt_validates_and_says_so_distinctly_from_missing() { + // "It was there and I could not read it" and "it was not there" both + // validate, but the operator must be able to tell them apart. + let d = decide(Err("permission denied".to_string()), &key(), false); + assert_eq!( + d, + F2Decision::Validate(F2ValidateReason::Unreadable( + "permission denied".to_string() + )) + ); + assert_ne!(d, F2Decision::Validate(F2ValidateReason::NoReceipt)); +} + +#[test] +fn a_receipt_from_an_older_schema_revalidates() { + let k = key(); + let mut old = receipt_for(&k); + old.schema = 0; + assert_eq!( + decide(Ok(Some(old)), &k, false), + F2Decision::Validate(F2ValidateReason::SchemaMismatch { + found: 0, + expected: F2_RECEIPT_SCHEMA + }) + ); +} + +// ---------------------------------------------------------------- done_when 2 +#[test] +fn revalidate_forces_a_fresh_run_even_on_a_perfect_receipt() { + let k = key(); + assert_eq!( + decide(Ok(Some(receipt_for(&k))), &k, true), + F2Decision::Validate(F2ValidateReason::Revalidate) + ); +} + +#[test] +fn revalidate_wins_over_every_other_reason_so_the_message_names_the_flag() { + // With --revalidate the user does not want to hear about a stale receipt; + // they asked for a run. The reason is the flag, whatever the file says. + let k = key(); + for found in [ + Ok(None), + Err("boom".to_string()), + Ok(Some({ + let mut r = receipt_for(&k); + r.key.device = "other".to_string(); + r + })), + ] { + assert_eq!( + decide(found, &k, true), + F2Decision::Validate(F2ValidateReason::Revalidate) + ); + } +} + +// ------------------------------------------------------------ the file layer +#[test] +fn a_written_receipt_reads_back_equal_and_a_missing_one_is_none_not_err() { + let dir = std::env::temp_dir().join(format!("f2-receipt-test-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let k = key(); + let path = receipt_path(&dir, &k.model_sha256); + + assert_eq!( + read_receipt(&path), + Ok(None), + "no file must read as None, not Err" + ); + + let r = receipt_for(&k); + write_receipt(&path, &r).expect("write"); + assert_eq!(read_receipt(&path), Ok(Some(r))); + // No temp file of any name survives a successful write. + let leftovers: Vec<_> = std::fs::read_dir(&dir) + .expect("dir") + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with(".tmp")) + .collect(); + assert!( + leftovers.is_empty(), + "temp files left beside the receipt: {leftovers:?}" + ); + + // Corrupt it: that is Err, and it names the file. + std::fs::write(&path, "{not json").expect("corrupt"); + let err = read_receipt(&path).expect_err("garbage must not parse"); + assert!(err.contains("not a receipt"), "{err}"); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_receipt_file_carries_all_three_keys_flat_so_a_human_can_read_them() { + let k = key(); + let json = serde_json::to_string(&receipt_for(&k)).expect("serialize"); + for needle in [ + "\"model_sha256\"", + "\"apr_version\"", + "\"device\"", + "\"schema\"", + "\"positions_judged\"", + ] { + assert!(json.contains(needle), "receipt JSON lacks {needle}: {json}"); + } +} + +#[test] +fn model_sha256_is_the_real_sha256_lowercase_hex() { + // sha256("") and sha256("abc") are the two everyone can check by hand. + assert_eq!( + model_sha256(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + model_sha256(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); +} + +#[test] +fn apr_version_is_this_crates_version() { + assert_eq!(apr_version(), env!("CARGO_PKG_VERSION")); + assert!(!apr_version().is_empty()); +} + +#[test] +fn two_writers_on_one_model_each_rename_a_whole_file() { + // The race lane 1 named: a shared `.json.tmp` let writer B truncate + // what writer A was about to rename. With per-writer temp names, whichever + // rename lands last, the receipt on disk is a COMPLETE receipt from one of + // them — never a partial. Exercised by racing threads through the same + // path many times and parsing the survivor every time. + let dir = std::env::temp_dir().join(format!("f2-receipt-race-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let k = key(); + let path = receipt_path(&dir, &k.model_sha256); + let mut a = receipt_for(&k); + a.positions_judged = 11; + let mut b = receipt_for(&k); + b.positions_judged = 22; + + for _ in 0..40 { + let (pa, pb, ra, rb) = (path.clone(), path.clone(), a.clone(), b.clone()); + let ta = std::thread::spawn(move || write_receipt(&pa, &ra)); + let tb = std::thread::spawn(move || write_receipt(&pb, &rb)); + ta.join().expect("thread a").expect("write a"); + tb.join().expect("thread b").expect("write b"); + let survivor = read_receipt(&path) + .expect("the receipt must always parse") + .expect("the receipt must exist"); + assert!( + survivor == a || survivor == b, + "the survivor is neither writer's whole receipt: {survivor:?}" + ); + } + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs index f46e6eb5b0..ec870db618 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs @@ -1429,7 +1429,12 @@ fn run_qwen35_generate_gpu( "Backend: GPU (CUDA, {device_name}, {vram_mb} MB VRAM) [qwen35 hybrid forward, #3090]" ); - if !f2_validate_qwen35(&mut gpu, &qwen, input_tokens) { + // #3604: the guard runs once per (model sha256, apr version, device) and + // leaves a receipt; a later run whose triple matches reads it instead of + // re-deriving a 64-position CPU forward that was 67 % of a 14 s TTFT. + let f2 = + f2_validate_qwen35_receipted(&mut gpu, &qwen, input_tokens, mapped.data(), &device_name); + if !f2.accepted { return Err("the F2 CPU-parity guard rejected the GPU path".to_string()); } qwen35_gpu_decode(&mut gpu, input_tokens, gen_config) @@ -1496,24 +1501,54 @@ fn qwen35_gpu_decode( /// /// Both states are throwaway: the guard allocates its own, and the generation /// that follows allocates another. +/// What the F2 guard concluded — and, separately, whether it concluded +/// anything at all. +/// +/// The distinction exists for the receipt (#3604): three of this guard's exits +/// return "let the GPU serve" WITHOUT having compared a single position — +/// `SKIP_PARITY_GATE=1`, a probe shorter than two tokens, a CPU reference that +/// would not run. A receipt written on any of those would let a one-token +/// prompt "validate" the (model, apr, device) triple for every prompt after it. +/// So the wrapper writes a receipt on `Accepted` only. +#[cfg(feature = "cuda")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum F2Verdict { + /// CPU and GPU agreed on `positions_judged` positions. + Accepted { + /// Probe positions plus the one greedy decode step. + positions_judged: usize, + }, + /// They disagreed, or the GPU probe failed: fail closed. + Rejected, + /// Nothing was compared. The GPU may serve, but nothing was proved. + NotJudged, +} + +#[cfg(feature = "cuda")] +impl F2Verdict { + const fn lets_the_gpu_serve(self) -> bool { + !matches!(self, Self::Rejected) + } +} + #[cfg(feature = "cuda")] fn f2_validate_qwen35( gpu: &mut crate::gguf::cuda::Qwen35CudaModel<'_>, cpu: &Qwen35Model<'_>, probe_context: &[u32], -) -> bool { +) -> F2Verdict { // Same escape hatch as the dense gate, and the same one `apr parity` uses. if std::env::var("SKIP_PARITY_GATE").is_ok_and(|v| v == "1") { - return true; + return F2Verdict::NotJudged; } let probe = &probe_context[probe_context.len().saturating_sub(QWEN35_F2_PROBE_MAX)..]; // A one-token probe has no REAL position (≥1) to judge; position 0 is the // context-less near-tie the dense gate excludes for the same reason. if probe.len() < 2 { - return true; + return F2Verdict::NotJudged; } let Some(cpu_per_pos) = f2_qwen35_cpu_reference(cpu, probe) else { - return true; // the CPU forward itself failed: nothing to judge against. + return F2Verdict::NotJudged; // the CPU forward itself failed: nothing to judge against. }; let decode_token = cpu_per_pos .get(probe.len().saturating_sub(1)) @@ -1522,18 +1557,153 @@ fn f2_validate_qwen35( Ok(v) => v, Err(msg) => { eprintln!("{msg}"); - return false; // fail closed. + return F2Verdict::Rejected; // fail closed. }, }; let report = crate::infer::f2_multi_position_report(&cpu_per_pos, &gpu_per_pos); if report.accepted { - true + F2Verdict::Accepted { + positions_judged: cpu_per_pos.len(), + } } else { eprintln!( "{}", crate::infer::f2_divergence_msg(&report, crate::infer::F2ProbePath::Serial) ); - false + F2Verdict::Rejected + } +} + +/// The outcome of the receipted guard, for the caller and — once #3606's +/// `StageTimings` lands — for `apr run --json`, where `source` is what makes +/// a cached validation distinguishable from a fresh one (#3604 `done_when` 5). +#[cfg(feature = "cuda")] +#[derive(Debug, Clone, PartialEq)] +pub struct F2Outcome { + /// May the GPU serve this run? + pub accepted: bool, + /// `"receipt"` when the forward was skipped on a matching receipt, + /// `"fresh"` when it ran, `"not-judged"` when it ran but compared nothing. + pub source: &'static str, + /// Wall time of the guard itself, EXCLUDING the hash. ~0 on a receipt hit. + pub validate_ms: f64, + /// Wall time of hashing the model file — the price of the receipt's key, + /// paid on every run, reported separately so it cannot hide in either + /// number above. + pub sha256_ms: f64, + /// Where the receipt was read from or written to, if a cache dir exists. + pub receipt_path: Option, +} + +/// [`f2_validate_qwen35`] behind its receipt (#3604). +/// +/// Validate once per (model sha256, apr version, device); later runs of the +/// same triple read the receipt and skip the forward; `--revalidate` forces a +/// fresh run and rewrites it. Every path that is not a three-key match +/// validates — see `f2_receipt.rs` for the table — and a receipt is written on +/// [`F2Verdict::Accepted`] only, never on a verdict that judged nothing. +/// +/// Nothing in here can fail the run except the guard's own rejection: an +/// unwritable cache directory is reported and the next run simply validates +/// again. +#[cfg(feature = "cuda")] +fn f2_validate_qwen35_receipted( + gpu: &mut crate::gguf::cuda::Qwen35CudaModel<'_>, + cpu: &Qwen35Model<'_>, + probe_context: &[u32], + model_bytes: &[u8], + device_name: &str, +) -> F2Outcome { + use crate::gguf::f2_receipt::{ + apr_version, decide, model_sha256, read_receipt, receipt_dir, receipt_path, + revalidate_requested, unix_now, write_receipt, F2Decision, F2Receipt, F2ReceiptKey, + F2_RECEIPT_SCHEMA, + }; + + let hash_start = std::time::Instant::now(); + let key = F2ReceiptKey { + model_sha256: model_sha256(model_bytes), + apr_version: apr_version(), + device: device_name.to_string(), + }; + let sha256_ms = hash_start.elapsed().as_secs_f64() * 1000.0; + + let path = receipt_dir().map(|d| receipt_path(&d, &key.model_sha256)); + let found = match path.as_deref() { + Some(p) => read_receipt(p), + None => Ok(None), + }; + + match decide(found, &key, revalidate_requested()) { + F2Decision::Skip { receipt } => { + let age_s = unix_now().saturating_sub(receipt.validated_at); + eprintln!( + "F2 guard: receipt matches (model sha256 {}…, apr {}, {}) — validated {}s ago on {} positions; CPU reference forward skipped [source=receipt, sha256 {:.0} ms]. `apr run --revalidate` forces a fresh run.", + &key.model_sha256[..12], + key.apr_version, + key.device, + age_s, + receipt.positions_judged, + sha256_ms + ); + return F2Outcome { + accepted: true, + source: "receipt", + validate_ms: 0.0, + sha256_ms, + receipt_path: path, + }; + }, + F2Decision::Validate(reason) => { + eprintln!("F2 guard: validating on this run ({reason}) [source=fresh]"); + }, + } + + let start = std::time::Instant::now(); + let verdict = f2_validate_qwen35(gpu, cpu, probe_context); + let validate_ms = start.elapsed().as_secs_f64() * 1000.0; + + let source = match verdict { + F2Verdict::Accepted { positions_judged } => { + match path.as_deref() { + Some(p) => { + let receipt = F2Receipt { + schema: F2_RECEIPT_SCHEMA, + key, + validated_at: unix_now(), + positions_judged, + }; + match write_receipt(p, &receipt) { + Ok(()) => eprintln!( + "F2 guard: passed in {validate_ms:.0} ms on {positions_judged} positions; receipt written to {} — the next run of this (model, apr, device) skips it.", + p.display() + ), + Err(e) => eprintln!( + "F2 guard: passed in {validate_ms:.0} ms, but the receipt could not be written ({e}); the next run validates again. Set APR_F2_RECEIPT_DIR to a writable directory." + ), + } + } + None => eprintln!( + "F2 guard: passed in {validate_ms:.0} ms; no cache directory (no HOME, XDG_CACHE_HOME or APR_F2_RECEIPT_DIR), so no receipt — every run validates." + ), + } + "fresh" + }, + F2Verdict::NotJudged => { + eprintln!( + "F2 guard: nothing was compared on this run (probe too short, SKIP_PARITY_GATE, or the CPU reference did not run); the GPU serves, and NO receipt is written." + ); + "not-judged" + }, + F2Verdict::Rejected => "fresh", + }; + + F2Outcome { + accepted: verdict.lets_the_gpu_serve(), + source, + validate_ms, + sha256_ms, + receipt_path: path, } } @@ -1620,8 +1790,7 @@ mod qwen35_route_tests { ); assert!( !notice.contains("#3090"), - "#3090 is the GPU forward, which now exists — citing it here is the \ - withdrawn 'the GPU does not implement it' notice: {notice}" + "#3090 is the GPU forward, which now exists — citing it here is the withdrawn 'the GPU does not implement it' notice: {notice}" ); } diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index 82e87b2530..7c6743e34f 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -124,6 +124,10 @@ mod quantized_tests; #[cfg(test)] mod tests; +/// #3604: the F2 hybrid guard's receipt. CUDA-free on purpose, so its decision +/// table is tested on every build. +#[path = "inference/forward/f2_receipt.rs"] +pub mod f2_receipt; /// Qwen3.5 / Qwen3.8 hybrid (Gated `DeltaNet` + gated attention) CPU forward (#3091). #[path = "inference/forward/forward_qwen35.rs"] pub mod forward_qwen35; diff --git a/docs/audits/impl-PMAT-3577-receipt.md b/docs/audits/impl-PMAT-3577-receipt.md new file mode 100644 index 0000000000..73d4c6bcbb --- /dev/null +++ b/docs/audits/impl-PMAT-3577-receipt.md @@ -0,0 +1,120 @@ +# PMAT-3577 / #3577 — receipts under contract: `parity-receipt-v2` + `extract:parity-receipt` + the back-fill + +`checkout-only: true` — `pv` is 0.65.2 fleet-wide and has neither `lint --gate` nor `extract`, so every +verdict below was rendered in a checkout at the HEAD-built `pv` 0.68.2. A verdict on ≥ 1 fleet host is owed +once the pin lands (#3567), and this row does not wait for it. + +Worktree `/mnt/nvme-raid0/agent-wt/rel-3577`, branch `PMAT-3577-receipts-under-contract`, cut from +`origin/main` 863ba48ef. Measurements re-run by me, not taken from a lane. + +## What the row found before it built anything + +**1. The committed denominator is 7, not 8.** The ticket and the cop both said 8 (7 legacy + the #3574 +receipt). Measured over `git ls-files 'evidence/parity/**/*.json'`: 113 files, **7** parity records, **0** +with a `comparator`. The #3574 receipt is in PR #3575, which is **still open** — it is not on `main`. The +cop confirmed and will bump the denominator to 8 as part of landing #3575, which is this row's own +falsifier firing on its first real use. + +**2. Item 6 of the ticket is mis-aimed, in the way the ticket itself warns about.** The ruling said +"`receipt-lint` is deleted, replaced by `pv lint`"; the ticket corrected that (`receipt-lint` is +paiml-implement's quorum linter) and re-aimed item 6 at `scripts/check_parity_receipt.sh`. Measured: that +script validates a **different artifact family**. Its fixtures carry `instrument`, `protocol_ref`, `lanes[]` +with `decode_tok_per_sec` and a llama.cpp `build_commit`; `bench_receipt.py::validate_parity` requires +`instrument`, `protocol_ref`, `model`, `lanes`. A logit record has never carried one of them. Its callers +are `check_perf_claims_cite_receipts.sh`, `check_multiplatform_dogfood.sh` and `parity_host_receipt.sh`. +Folding it in would have deleted the validator for #2696 — published-apr-takes-the-CPU-path-and-reports-0.099x +— from a family nobody was watching. **`check_parity_receipt.sh` is out of scope and untouched**, agreed with +the cop, and the `done_when` line is satisfied by the honest answer: nothing to fold, different family. + +The ruling's premise gets *stronger*: "receipts are the only artifact family with no shape" is **literally +true** of the logit records. No validator of any kind, ever. That is why seven of them carried no comparator +for months — there was nothing that could have noticed. + +**3. `model_sha256` is `pattern` with no `minCount`, deliberately.** Six of the seven never recorded a model +hash. Hashing the files on the hosts today and attaching that to a receipt about 2026-09-06 would be a claim +about a different world wearing a witness's clothes. Absent means no measurement, never a match (ONT-4c1); +`partially_receipted: true` and an `unmeasured` entry carry the honesty. Confirmed by the cop before use. + +## What landed + +| | | +|---|---| +| `contracts/parity-receipt-v2.yaml` | 3 shapes: `parity-receipt-complete` (closed, `ignoredProperties: []`), `parity-comparator-self`, `parity-comparator-oracle` | +| `contracts/parity-receipt-v1.yaml` | the retired layout, recorded; **no shape** — see below | +| `crates/aprender-contracts/src/ontology/extract/parity_receipt.rs` | the extractor + 10 unit cases | +| `scripts/parity_receipt_denominator.sh` | the INDEPENDENT predicate + 4-case self-test | +| `evidence/parity/EXPECTED_RECEIPTS` | the committed denominator (7) | +| `crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs` | 8 CLI cases over 5 fixtures | +| `evidence/parity/**` (7 files) | migrated to v2 + back-filled, one commit | +| `contracts/{contracts.nt,shapes.ttl,census.json}` | regenerated (derived, R-18 / decision 8) | + +**v1 declares no shape, on purpose.** All seven instances were migrated, so a shape over the retired layout +would target a class nothing instantiates and pass vacuously — worth less than no shape. The enforcement that +replaces it fires: the extractor **refuses** an unmigrated record by name, and the denominator predicate +refuses it independently. + +**Nothing is armed.** Arming lives in `contracts/lint-baseline.json` `armed_shapes[]`, a shared file this row +is forbidden to touch (decision 7). The three shapes are computed and reported, exactly as `ladder-green` was +at ONT-4c1; arming them is a follow-up carrying the `touches-shared-contracts` label. + +## Controls, every one measured in both directions + +| control | before | after | +|---|---|---| +| **the back-fill** — comparator stripped from all 7 vs as committed | **7** parity-shape violations | **0** | +| **the plant** — comparator removed from one record | **exactly 1**, naming `ont:parity/comparator` and `minCount` | 0 on restore | +| **the mutation** — `in:` widened to accept `oracle`, contract + all 5 fixtures | `ont4c3_parity_receipts` **FAILS** (`a_comparator_kind_the_shape_does_not_accept_fails`) | 8/8 on restore | +| **the mutation, other half** — only the real contract widened | **FAILS** (`every_fixture_carries_the_real_contract_byte_for_byte`) | 8/8 on restore | +| **an unmigrated record** (hit for real when a `git checkout` reverted the migration mid-run) | `Unknown{WrongCorpus}`, **exit 2**, each legacy file refused by name | Pass after re-migration | +| **denominator drift** — 2 receipts, committed 1 | exit 2, "matched 2 … says 1" | — | +| **the three answers are distinct** | `parity-green` 0 · `parity-nocomparator` 1 · `parity-unmigrated` 2 | — | + +``` +cargo test -p aprender-contracts --lib ontology::extract::parity_receipt 10 passed +cargo test -p aprender-contracts --lib ontology::verdict 8 passed +cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts 8 passed +bash scripts/parity_receipt_denominator.sh --self-test 4 ok +bash scripts/parity_receipt_denominator.sh PASS 7 receipt(s), EXPECTED_RECEIPTS says 7 +pv lint contracts --gate shapes Pass, violations 0, parity-shape violations 0 +pv extract contracts --check rc 0 +``` + +`by_shape` on the committed tree: `parity-receipt-complete=7`, `parity-comparator-self=7`, +`parity-comparator-oracle=0`. + +## The back-fill is a relabel, and every field cites committed evidence + +`raw` in each migrated record is the original `apr parity --json` document, key for key. The envelope was +quoted from files already in the tree, named in each record's `provenance.record`: +`evidence/parity/l0-1/{lambda,gx10}/RECORD.md` and `evidence/parity/l0-1b/gx10/n5/DETERMINISM.md`. + +**One derived field was wrong on the first pass and is worth recording.** `result.verdict` initially copied +`raw.parity` — apr's own per-position band flag — which says PASS for the two 1.5B cells that their own +`RECORD.md` calls RED. It now resolves the threshold from `evidence/parity/thresholds.yaml` (`default.min_cosine` +0.98, `min_positions` 64) and judges min cosine over ≥ 64 positions, which reproduces all seven readings the +RECORD.md files state, including both REDs. A threshold is **never** typed into the shape — that is this row's +STOP condition — and `thresholdSource` is `resolves:`, with a missing path materialised as +`thresholdSourceMissing` for a shape to refuse with `maxCount 0`. + +Cross-check: the migrated `min_cosine` values reproduce `evidence/parity/thresholds.yaml`'s basis text to six +decimals (7B lambda 0.998607, 7B gx10 0.998465, 1.5B lambda 0.950827, 1.5B gx10 0.950611). + +## Not done, and why + +**`ONT-4c3 bound in the ONT-001 ledger` — cannot be done from this repository.** The ledger is +`docs/specifications/paiml-ontology.md` in **paiml/infra**, where v4.8 defines ONT-4c3 as **kernel** receipts +(`entity type kernel`, `apr-kernel-receipt/v1`, the `kernel-parity`/`kernel-timing`/`kernel-safety` shapes). +**Filed as paiml/infra#814** with both resolutions (renumber the parity row, or record the re-scope in a +v4.9) and no preference between them — the point is that one identifier should not mean two things in two +repositories. The re-scope is an aprender-side ruling that the infra spec does not yet carry, so binding +it is an **infra PR**, not this one. Raised with the cop rather than left as a checked box. The kernel +sub-row with the `gated_rmsnorm` fixture stays a follow-up either way. + +## Follow-ups + +- Arm the three shapes in `contracts/lint-baseline.json` (`touches-shared-contracts`, group of one). +- Bind ONT-4c3 in infra's ONT-001 ledger once **paiml/infra#814** decides which meaning the identifier keeps. +- #3575 bumps `EXPECTED_RECEIPTS` to 8 when it lands, and its receipt needs the v2 envelope + (`comparator`, `partially_receipted`, `backend`, `generated_at`) or these shapes will report it. +- `Receipt` as a shared parent class once quorum and dispatch receipts join (NOT this row). +- The dense oracle re-measurement, comparator ruling item (d). diff --git a/docs/audits/quorum-PMAT-3346.json b/docs/audits/quorum-PMAT-3346.json new file mode 100644 index 0000000000..a890916309 --- /dev/null +++ b/docs/audits/quorum-PMAT-3346.json @@ -0,0 +1,211 @@ +{ + "ticket": "PMAT-3346", + "base": "main", + "base_resolved": "origin/main", + "base_note": "local main differs from origin/main by 127 commit(s); judged against origin/main", + "head": "e060c7e2c22bc221b5e454a9b677d09d2b27c2e2", + "diff_sha256": "a9a91b0c1f95b9e18fb6a00b117963990981c632db99f3a4b240f13a321a81ab", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 48102, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "I have reviewed the diff against the requirements specified in PMAT-3346. I applied the patch and ran `cargo test -p aprender-core` on the patched workspace. The tests passed, confirming that the arithmetic models both the Gated DeltaNet layer (21,555,360) and full-attention layer (18,352,640) correctly to hit the exact 752,393,024 parameter mark of Qwen3.5-0.8B-Q4_K_M.gguf. The 9B config accurately computes to 8.345B, appropriately documenting the gap and NOT forcefully discharging the constraint. The `binding.yaml` remains valid and unmodified where it matters (156 items). No gates were weakened and all receipt claims are backed by the diff. Therefore, the verdict is PASS.", + "findings": [ + { + "claim": "ModelConstraints carries inner_size, state_size, conv_kernel, group_count, full_attention_interval", + "file": "crates/aprender-core/src/format/model_family.rs", + "grounding": "cited", + "line": 250 + }, + { + "claim": "The config-derived model_parameter_count equals the measured GGUF tensor sum of a REAL file (320 tensors, 752,393,024 parameters, delta 0) with both gated-DeltaNet and full-attention layers matching exactly.", + "command": "cd pmat-3477/aprender && cargo test -p aprender-core", + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "grounding": "measured", + "line": 242, + "output": "running 280 tests... test qwen35_0_8b_config_derived_count_equals_the_measured_gguf_inventory ... ok" + }, + { + "claim": "The two shapes that contradict dense accounting are modelled from the tensors, not assumed: attn_q is [d, 2*n_h*d_k] (q projection emits output gate) and the DeltaNet mixer projections are counted for 18-of-24 linear layers.", + "file": "crates/aprender-core/src/format/model_arithmetic.rs", + "grounding": "cited", + "line": 121 + }, + { + "claim": "The 9B range instantiation is NOT asserted in this PR; the descriptor disagrees with itself so P(9B) computes to 8.345B, and settling it needs a real Qwen3.5-9B GGUF.", + "command": "cd pmat-3477/aprender && cargo test -p aprender-core", + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "grounding": "measured", + "line": 389, + "output": "running 280 tests... test qwen35_9b_hybrid_layers_still_fall_short_of_the_invariant_range ... ok" + }, + { + "claim": "The note edit leaves contracts/binding.yaml valid YAML with 156 entries.", + "command": "cat pmat-3477/aprender/contracts/binding.yaml | grep \"^-\" | wc -l", + "file": "contracts/binding.yaml", + "grounding": "measured", + "line": 764, + "output": "156" + } + ], + "raw_bytes": 7208, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the requested ticket criteria. The ModelConstraints now carry the delta net shape. The tests verify the GGUF arithmetic perfectly against the given Qwen3.5-0.8B parameters. The 9B range instantiation is left unverified and noted in bindings.yaml. The diff is fully backed by code logic and tests.", + "findings": [], + "raw_bytes": 3680, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly resolves PMAT-3346 by adding DeltaNetShape to ModelConstraints, updating build codegen and runtime YAML parsing to preserve the Gated DeltaNet shape parameters, and modeling per-layer parameter arithmetic for hybrid Gated DeltaNet architectures. Parameter accounting was empirically verified against the 320-tensor inventory of Qwen3.5-0.8B-Q4_K_M.gguf (752,393,024 parameters, delta 0). All unit tests and contract falsification tests pass. No gates were weakened, no tests assert opposite behavior, and no out-of-scope changes were made.", + "findings": [ + { + "claim": "ModelConstraints now carries DeltaNetShape (inner_size, state_size, conv_kernel, group_count, full_attention_interval) and config-derived parameter count matches the measured 752,393,024 parameters of Qwen3.5-0.8B-Q4_K_M.gguf with 0 delta. All 29 model_arithmetic unit tests pass.", + "command": "CARGO_TARGET_DIR=/tmp/target_3346 cargo test --lib -p aprender-core model_arithmetic", + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "fix": "Verified passing test suite", + "grounding": "measured", + "line": 198, + "output": "test result: ok. 29 passed; 0 failed; 0 ignored; 0 measured; 14245 filtered out; finished in 0.01s" + }, + { + "claim": "Contract falsification test falsify_mf_qwen35_010_deltanet_shape_reaches_constraints verifies qwen3_5 loads deltanet shape into ModelConstraints while all other families receive None.", + "command": "CARGO_TARGET_DIR=/tmp/target_3346 cargo test --lib -p aprender-core falsify_mf_qwen35_010_deltanet_shape_reaches_constraints", + "file": "crates/aprender-core/src/format/model_family_contract_falsify.rs", + "fix": "Verified passing contract test", + "grounding": "measured", + "line": 885, + "output": "test format::model_family_loader::contract_falsification::falsify_mf_qwen35_010_deltanet_shape_reaches_constraints ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 14273 filtered out; finished in 0.01s" + } + ], + "raw_bytes": 6032, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "contracts/binding.yaml", + "line": 764, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The note edit leaves contracts/binding.yaml valid YAML with 156 entries." + ] + }, + { + "file": "crates/aprender-core/src/format/model_arithmetic.rs", + "line": 121, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The two shapes that contradict dense accounting are modelled from the tensors, not assumed: attn_q is [d, 2*n_h*d_k] (q projection emits output gate) and the DeltaNet mixer projections are counted for 18-of-24 linear layers." + ] + }, + { + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "line": 198, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "ModelConstraints now carries DeltaNetShape (inner_size, state_size, conv_kernel, group_count, full_attention_interval) and config-derived parameter count matches the measured 752,393,024 parameters of Qwen3.5-0.8B-Q4_K_M.gguf with 0 delta. All 29 model_arithmetic unit tests pass." + ] + }, + { + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "line": 242, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The config-derived model_parameter_count equals the measured GGUF tensor sum of a REAL file (320 tensors, 752,393,024 parameters, delta 0) with both gated-DeltaNet and full-attention layers matching exactly." + ] + }, + { + "file": "crates/aprender-core/src/format/model_arithmetic_tests.rs", + "line": 389, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The 9B range instantiation is NOT asserted in this PR; the descriptor disagrees with itself so P(9B) computes to 8.345B, and settling it needs a real Qwen3.5-9B GGUF." + ] + }, + { + "file": "crates/aprender-core/src/format/model_family.rs", + "line": 250, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "ModelConstraints carries inner_size, state_size, conv_kernel, group_count, full_attention_interval" + ] + }, + { + "file": "crates/aprender-core/src/format/model_family_contract_falsify.rs", + "line": 885, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Contract falsification test falsify_mf_qwen35_010_deltanet_shape_reaches_constraints verifies qwen3_5 loads deltanet shape into ModelConstraints while all other families receive None." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3351.json b/docs/audits/quorum-PMAT-3351.json new file mode 100644 index 0000000000..af462e62a7 --- /dev/null +++ b/docs/audits/quorum-PMAT-3351.json @@ -0,0 +1,134 @@ +{ + "ticket": "PMAT-3351,PMAT-3338", + "base": "main", + "base_resolved": "origin/main", + "base_note": "local main differs from origin/main by 128 commit(s); judged against origin/main", + "head": "d915b8f16d8e6e6ca71f1cb80174ee656934cda3", + "diff_sha256": "91f1a74444bfab677fecdbe7149ffe30cfd7872203e8f67cc540044b7a14930b", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 47132, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff thoroughly addresses both PMAT-3338 and PMAT-3351. \n\n1. **PMAT-3338**: The fix in `truncate()` correctly walks back to the nearest char boundary (`!s.is_char_boundary(end)`) instead of slicing directly by byte index, eliminating the panic when encountering multi-byte characters like `∈`. The fix is backed up by `truncate_cuts_on_a_char_boundary_not_a_byte` test.\n2. **PMAT-3351**: The L2 column verification has been strengthened to check explicit links via `obligation`, `binds_to`, and `discharged_by` instead of relying on an index count. The `L2Status` enum correctly distinguishes between `Tested`, `Untested`, and `Unknown` depending on whether a linkage exists and can be resolved. \n3. **PMAT-3351 (Refusal)**: The `lint` command explicitly refuses `--strict-test-binding` on single files, correctly closing the loop for the false negative gate behavior where missing cross-references were unavoidable.\n\nNo gates were weakened; they were definitively strengthened. Both problems are fixed precisely as requested.", + "findings": [], + "raw_bytes": 4219, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff accurately implements the requirements for both PMAT-3351 and PMAT-3338.\n\n1. **PMAT-3351**:\n - The L2 column now correctly reads the actual obligation-to-test links (`discharged_by`, `obligation`/`binds_to`) instead of using an index comparison, correctly reporting `Tested`, `Untested`, or `Unknown`.\n - Single-file `pv lint --strict-test-binding` is now properly refused with a clear error message instructing the user to run the directory form instead of reporting a false negative.\n\n2. **PMAT-3338**:\n - The `truncate` function in `obligation_matrix.rs` now properly cuts strings at a character boundary rather than blindly at a byte index, preventing panics on multi-byte characters.\n\nNo unrequested changes were made, no gates were weakened, and all new behavior is well-tested.", + "findings": [], + "raw_bytes": 3781, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff fully satisfies the requirements of tickets PMAT-3351 (#3347) and PMAT-3338 (#3338).\n1. PMAT-3351: The L2 column in obligation_matrix.rs now inspects actual obligation<->test links (supporting discharged_by, obligation, binds_to, comma-separated lists, and YAML sequences via the Citation enum) instead of doing an index comparison (idx < falsification_tests.len()). Unlinked obligations evaluate to Unknown ('?'), contracts without tests evaluate to Untested ('✗'), and linked obligations evaluate to Tested ('✓'). Single-file `pv lint --strict-test-binding` is cleanly refused to prevent false negative reports when source indexing is unavailable.\n2. PMAT-3338: `truncate` in obligation_matrix.rs now steps back to the nearest UTF-8 character boundary (`s.is_char_boundary(end)`), preventing byte-slice panics on multi-byte characters (such as '∈') during table rendering.\n3. Quality & Safety: All gates are preserved or strengthened, unit and integration tests explicitly exercise both bug fixes and edge cases, and no extraneous changes were introduced.", + "findings": [ + { + "claim": "The L2 column reads obligation-to-test links (discharged_by and obligation/binds_to) rather than an index comparison, correctly distinguishing Tested, Untested, and Unknown statuses.", + "file": "crates/aprender-contracts/src/obligation_matrix.rs", + "grounding": "cited", + "line": 73 + }, + { + "claim": "Single-file pv lint --strict-test-binding is refused with an informative error rather than emitting false negative dangling reference warnings.", + "file": "crates/aprender-contracts-cli/src/commands/lint.rs", + "grounding": "cited", + "line": 55 + }, + { + "claim": "truncate cuts on UTF-8 char boundaries rather than byte indices, preventing panics on multi-byte characters in obligation properties.", + "file": "crates/aprender-contracts/src/obligation_matrix.rs", + "grounding": "cited", + "line": 318 + } + ], + "raw_bytes": 5941, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-contracts-cli/src/commands/lint.rs", + "line": 55, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Single-file pv lint --strict-test-binding is refused with an informative error rather than emitting false negative dangling reference warnings." + ] + }, + { + "file": "crates/aprender-contracts/src/obligation_matrix.rs", + "line": 73, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The L2 column reads obligation-to-test links (discharged_by and obligation/binds_to) rather than an index comparison, correctly distinguishing Tested, Untested, and Unknown statuses." + ] + }, + { + "file": "crates/aprender-contracts/src/obligation_matrix.rs", + "line": 318, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "truncate cuts on UTF-8 char boundaries rather than byte indices, preventing panics on multi-byte characters in obligation properties." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3401.json b/docs/audits/quorum-PMAT-3401.json new file mode 100644 index 0000000000..3378c689aa --- /dev/null +++ b/docs/audits/quorum-PMAT-3401.json @@ -0,0 +1,160 @@ +{ + "ticket": "PMAT-3401", + "base": "main", + "base_resolved": "origin/main", + "base_note": "local main differs from origin/main by 128 commit(s); judged against origin/main", + "head": "9f59bebb65d51481fef673857209e3e32063a16c", + "diff_sha256": "bb44e1f9146e1711a4fd2cc58ef708592dcf2c58dcd76c41174c84b7c9a38144", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 264947, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff exactly matches the ticket PMAT-3401 requirements. It introduces 24 DRAFT contracts for AutoGluon (crux-O-01 through 24) with LIVE-PENDING gates and no associated code implementations, CLI changes, or test bodies. It correctly increments CRUX_COMPETITORS in validator.rs from 14 to 15, adds autogluon to both lists in crux_intake_tests.rs, and introduces the category O in the crux_scaffold_contracts.py script. The master registry and specifications (v2.3) have been appropriately updated, including evidence_sources.autogluon, 24 new stories, and modifications in §3, §5, and §6 of the CRUX workflows spec. It also correctly supplies the 4 evidence files. Furthermore, the roadmap components are properly configured (26 fragments spanning PMAT-3370, 3371-3394, and 3401) with a regenerated roadmap.yaml. Lastly, the three corpus derivatives (census.json, contracts.nt, README CONTRACT_COUNT) are all correctly updated reflecting the addition of 24 contracts (1800 -> 1824). I found no refutations. PASS.", + "findings": [], + "raw_bytes": 4247, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly matches all the requirements from the ticket. It adds exactly 24 DRAFT contracts (crux-O-01 to crux-O-24) with LIVE-PENDING gates, updates CRUX_COMPETITORS to 15 in validator.rs and crux_intake_tests.rs, adds the category O in the python script, adds the master-registry rows and evidence sources, updates the specification document for v2.3, adds the 4 evidence files, adds the 26 roadmap fragments and regenerates roadmap.yaml, and correctly updates the three corpus derivatives to reflect the 24 new contracts. No unintended changes or unrequested implementations were found.", + "findings": [], + "raw_bytes": 3346, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 review of PMAT-3401 complete. The diff admits AutoGluon 1.6.3 to CRUX as Category O with 24 draft contracts (crux-O-01..24) featuring LIVE-PENDING gates, CRUX_COMPETITORS 14->15 in validator.rs and crux_intake_tests.rs, CATEGORY_NAMES[O] in crux_scaffold_contracts.py, 24 master-registry rows + evidence_sources.autogluon in crux-competitive-research-ux-v1.yaml, workflow spec updates v2.3, 4 evidence files, 26 roadmap fragments + regenerated roadmap.yaml, and all 3 corpus derivatives (census.json, contracts.nt, README.md). All 1666 tests pass, pv census and extract pass with zero drift, and no refutations were found. Verdict: PASS.", + "findings": [ + { + "claim": "All 1666 tests in aprender-contracts pass cleanly with zero failures.", + "command": "cargo test -p aprender-contracts", + "file": "crates/aprender-contracts/src/schema/validator.rs", + "fix": "None required; all tests pass.", + "grounding": "measured", + "line": 120, + "output": "test result: ok. 1666 passed; 0 failed; 5 ignored; 0 measured; 0 filtered out; finished in 23.64s" + }, + { + "claim": "contracts/census.json matches the output of fresh pv census for 1824 contracts.", + "command": ". scripts/pv_bin.sh && \"$PV\" census contracts --format json > /tmp/test_census.json && diff -u contracts/census.json /tmp/test_census.json", + "file": "contracts/census.json", + "fix": "None required; census is fully synchronized.", + "grounding": "measured", + "line": 5, + "output": "diff exit code 0, 1824 contracts counted in contracts/census.json" + }, + { + "claim": "README.md CONTRACT_COUNT blocks are synchronized to 1824 contracts.", + "command": "bash scripts/readme_sync.sh --check", + "file": "README.md", + "fix": "None required; README matches measured contract count.", + "grounding": "measured", + "line": 25, + "output": "ok readme_sync: README.md already states the measured count 1824 in every CONTRACT_COUNT block" + }, + { + "claim": "The 24 category O contracts crux-O-01..24 were added with draft status, LIVE-PENDING gates, and no implementation as required by PMAT-3401.", + "file": "contracts/crux-O-01-v1.yaml", + "fix": "None required; contracts meet specification.", + "grounding": "cited", + "line": 202 + } + ], + "raw_bytes": 6493, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "README.md", + "line": 25, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "README.md CONTRACT_COUNT blocks are synchronized to 1824 contracts." + ] + }, + { + "file": "contracts/census.json", + "line": 5, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "contracts/census.json matches the output of fresh pv census for 1824 contracts." + ] + }, + { + "file": "contracts/crux-O-01-v1.yaml", + "line": 202, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The 24 category O contracts crux-O-01..24 were added with draft status, LIVE-PENDING gates, and no implementation as required by PMAT-3401." + ] + }, + { + "file": "crates/aprender-contracts/src/schema/validator.rs", + "line": 120, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "All 1666 tests in aprender-contracts pass cleanly with zero failures." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3496.json b/docs/audits/quorum-PMAT-3496.json new file mode 100644 index 0000000000..75f334ebef --- /dev/null +++ b/docs/audits/quorum-PMAT-3496.json @@ -0,0 +1,180 @@ +{ + "ticket": "PMAT-3496", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "897574628b5233a8292e1f0ddc9e613685e69c83", + "diff_sha256": "52ec095a43bb1195ade36a752d71b633f8a5f90d1f39329477ef90ea93e4acfe", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 5239, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements the acceptance criteria for PMAT-3496. It creates `docs/roadmaps/entries/PMAT-3495.yaml` and `docs/roadmaps/entries/PMAT-3496.yaml` with the precise titles specified in the ticket. It also adds these entries to `docs/roadmaps/roadmap.yaml`. The diff is strictly additive and touches nothing outside of `docs/roadmaps/`, with no unauthorized changes (e.g. no Kani harnesses or CI wiring included). No flaws or regressions were found.", + "findings": [], + "raw_bytes": 3186, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements PMAT-3496 by registering PMAT-3495 and PMAT-3496 in the roadmap. The titles are transcribed verbatim, and the diff only touches the permitted roadmap files.", + "findings": [ + { + "claim": "The title for PMAT-3495 is transcribed verbatim as required by the acceptance criteria of PMAT-3496.", + "file": "docs/roadmaps/entries/PMAT-3495.yaml", + "grounding": "cited", + "line": 4 + }, + { + "claim": "The title for PMAT-3496 matches the ticket's title exactly.", + "file": "docs/roadmaps/entries/PMAT-3496.yaml", + "grounding": "cited", + "line": 4 + }, + { + "claim": "The roadmap.yaml file is additively updated with the new entries, matching the individual files.", + "file": "docs/roadmaps/roadmap.yaml", + "grounding": "cited", + "line": 18631 + } + ], + "raw_bytes": 3577, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 Reviewer verdict: PASS. The diff registers PMAT-3495 and PMAT-3496 in the roadmap entries directory and additively updates docs/roadmaps/roadmap.yaml. All acceptance criteria are satisfied: PMAT-3495 title matches issue #3495 verbatim, the roadmap YAML changes are purely additive (0 deletions, 0 reserialisations), valid YAML syntax is maintained, and no files outside docs/roadmaps/ are touched.", + "findings": [ + { + "claim": "docs/roadmaps/entries/PMAT-3495.yaml creates entry PMAT-3495 with issue 3495 and exact title matching issue #3495 verbatim", + "file": "docs/roadmaps/entries/PMAT-3495.yaml", + "fix": "N/A", + "grounding": "cited", + "line": 1 + }, + { + "claim": "docs/roadmaps/entries/PMAT-3496.yaml creates entry PMAT-3496 with issue 3495 and exact title matching ticket PMAT-3496 verbatim", + "file": "docs/roadmaps/entries/PMAT-3496.yaml", + "fix": "N/A", + "grounding": "cited", + "line": 1 + }, + { + "claim": "docs/roadmaps/roadmap.yaml additively aggregates entries PMAT-3495 and PMAT-3496 with 0 deletions or reserialisations and valid YAML syntax", + "command": "python3 -c \"import yaml; diff_pmat3495 = '''- id: PMAT-3495\\n github_issue: 3495\\n item_type: task\\n title: 'VERIFY-001: run the 132 Kani harnesses in CI (proof credit from runs, not declarations) and pilot Verus on one dequant/parser function'\\n status: planned\\n priority: medium\\n assigned_to: null\\n created: 2026-09-18T16:43:48Z\\n updated: 2026-09-18T16:43:48Z\\n spec: null\\n acceptance_criteria: []\\n phases: []\\n subtasks: []\\n estimated_effort: null\\n labels: []\\n notes: null'''; diff_pmat3496 = '''- id: PMAT-3496\\n github_issue: 3495\\n item_type: task\\n title: 'Register PMAT-3495 (VERIFY-001 on 0.71.0) in the roadmap so the AD-04 quorum for PR #3496 can run (registration only)'\\n status: planned\\n priority: medium\\n assigned_to: null\\n created: 2026-09-21T01:39:59Z\\n updated: 2026-09-21T01:39:59Z\\n spec: null\\n acceptance_criteria: []\\n phases: []\\n subtasks: []\\n estimated_effort: null\\n labels:\\n - kind:triage\\n notes: \\\"ACCEPTANCE...\\\"'''; parsed_3495 = yaml.safe_load(diff_pmat3495); parsed_3496 = yaml.safe_load(diff_pmat3496); assert parsed_3495[0]['id'] == 'PMAT-3495'; assert parsed_3496[0]['id'] == 'PMAT-3496'; print('YAML validation passed cleanly!')\"", + "file": "docs/roadmaps/roadmap.yaml", + "fix": "N/A", + "grounding": "measured", + "line": 18631, + "output": "no git remotes found\nYAML validation passed cleanly!" + } + ], + "raw_bytes": 7249, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "docs/roadmaps/entries/PMAT-3495.yaml", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "docs/roadmaps/entries/PMAT-3495.yaml creates entry PMAT-3495 with issue 3495 and exact title matching issue #3495 verbatim" + ] + }, + { + "file": "docs/roadmaps/entries/PMAT-3495.yaml", + "line": 4, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The title for PMAT-3495 is transcribed verbatim as required by the acceptance criteria of PMAT-3496." + ] + }, + { + "file": "docs/roadmaps/entries/PMAT-3496.yaml", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "docs/roadmaps/entries/PMAT-3496.yaml creates entry PMAT-3496 with issue 3495 and exact title matching ticket PMAT-3496 verbatim" + ] + }, + { + "file": "docs/roadmaps/entries/PMAT-3496.yaml", + "line": 4, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The title for PMAT-3496 matches the ticket's title exactly." + ] + }, + { + "file": "docs/roadmaps/roadmap.yaml", + "line": 18631, + "lanes_agreeing": [ + 2, + 3 + ], + "claims": [ + "The roadmap.yaml file is additively updated with the new entries, matching the individual files.", + "docs/roadmaps/roadmap.yaml additively aggregates entries PMAT-3495 and PMAT-3496 with 0 deletions or reserialisations and valid YAML syntax" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3577.json b/docs/audits/quorum-PMAT-3577.json new file mode 100644 index 0000000000..33f19cdc89 --- /dev/null +++ b/docs/audits/quorum-PMAT-3577.json @@ -0,0 +1,288 @@ +{ + "ticket": "PMAT-3577", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "445224d9ed3509f8e938414933ad4948ffcce6d5", + "diff_sha256": "3eaaa279c9ff352034e9b3782a60e04be118337a00a02be37d76ef1d75ab6bac", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 683926, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly satisfies the requirements of PMAT-3577. It introduces the v2 shape for parity receipts (with thresholds correctly resolved via path rather than typed into the shape), fully skips adding a shape for v1 (in compliance with the 'do NOT add a v1 shape' STOP condition), updates the EXPECTED_RECEIPTS exactly to the back-fill denominator of 7, and explicitly documents why out-of-scope validations (e.g. check_parity_receipt.sh) are untouched. No weakened gates or unpermitted shared file modifications were found. The implementation strictly adheres to the stated ticket guidelines.", + "findings": [ + { + "claim": "The v1 receipt layout declares no shape, strictly abiding by the STOP condition 'do NOT add a v1 shape' as described in the amendment.", + "file": "contracts/parity-receipt-v1.yaml", + "grounding": "cited", + "line": 31 + }, + { + "claim": "The shape dynamically resolves the threshold via thresholdSource using 'resolves: path' instead of hardcoding any threshold value, satisfying the STOP condition.", + "file": "contracts/parity-receipt-v2.yaml", + "grounding": "cited", + "line": 55 + }, + { + "claim": "The denominator count for EXPECTED_RECEIPTS is correctly pinned to 7, exactly matching the number of back-filled receipts.", + "file": "evidence/parity/EXPECTED_RECEIPTS", + "grounding": "cited", + "line": 9 + } + ], + "raw_bytes": 4837, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff strictly implements the amended PMAT-3577 requirements. It introduces the parity-receipt-v2 shape, implements the extract:parity-receipt logic, successfully back-fills the 7 legacy receipts, and leaves the v1 contract intentionally shapeless as required by the amendment. The PR properly refrains from arming the shapes in `lint-baseline.json` and accurately updates only the in-scope `check_model_parity.sh`. All claims in the receipt accurately reflect the diff, tests pass successfully, and no gates are weakened (the `shapes_gate` is appropriately strengthened to decline on `WrongCorpus`).", + "findings": [ + { + "claim": "The diff correctly leaves parity-receipt-v1 shapeless in accordance with the amendment of PMAT-3577.", + "file": "contracts/parity-receipt-v1.yaml", + "grounding": "cited" + }, + { + "claim": "The denominator predicate correctly expects and matches 7 receipts.", + "command": "bash scripts/parity_receipt_denominator.sh", + "file": "evidence/parity/EXPECTED_RECEIPTS", + "grounding": "measured", + "output": "PASS 7 receipt(s), EXPECTED_RECEIPTS says 7" + }, + { + "claim": "The check_parity_receipt.sh script was left untouched.", + "command": "git diff --name-only origin/main...HEAD | grep check_parity_receipt.sh || echo \"Not touched\"", + "file": "scripts/check_parity_receipt.sh", + "grounding": "measured", + "output": "Not touched" + }, + { + "claim": "The gate shapes_gate was not weakened; it was actually strengthened to return WrongCorpus on a mismatch.", + "file": "crates/aprender-contracts/src/lint/shapes_gate.rs", + "grounding": "cited" + }, + { + "claim": "The tests for ont4c3 parity receipts pass, including the negative condition for a comparator kind the shape does not accept.", + "command": "cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts", + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "grounding": "measured", + "output": "running 9 tests\\ntest every_fixture_carries_the_real_contract_byte_for_byte ... ok\\ntest an_unmigrated_legacy_record_declines_with_exit_2_and_is_named ... ok\\ntest a_receipt_added_without_bumping_the_denominator_declines_naming_both_numbers ... ok\\ntest a_receipt_with_no_comparator_fails_naming_the_property ... ok\\ntest a_complete_receipt_passes_with_one_focus_node ... ok\\ntest a_comparator_kind_the_shape_does_not_accept_fails ... ok\\ntest the_three_answers_are_distinct ... ok\\ntest the_committed_tree_agrees_with_its_own_denominator ... ok\\ntest the_entity_type_is_counted_in_by_entity_type_not_merely_registered ... ok\\n\\ntest result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 8.13s" + } + ], + "raw_bytes": 8765, + "err_bytes": 59, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed ticket PMAT-3577, receipt PMAT-3577, refutation doctrine, and full diff in /mnt/nvme-raid0/agent-wt/rel-3577/docs/audits/quorum-PMAT-3577.prompt.md. Confirmed that parity-receipt-v2.yaml defines the 3 required shapes, parity-receipt-v1.yaml documents the superseded layout, extract:parity-receipt extracts logit-parity records into RDF focus nodes while refusing unmigrated records by name, the independent denominator predicate script passes all self-tests, all 7 legacy receipts under evidence/parity/** are back-filled to v2, and all unit and CLI integration tests pass cleanly without weakening any gates. Verdict: PASS.", + "findings": [ + { + "claim": "extract:parity-receipt unit tests pass cleanly with 10 passed tests, verifying classification, legacy record refusal by name, missing threshold detection, positive control, and denominator checks.", + "command": "cargo test -p aprender-contracts --lib ontology::extract::parity_receipt", + "file": "crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs", + "grounding": "measured", + "line": 1, + "output": "test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 1671 filtered out; finished in 0.01s" + }, + { + "claim": "All 9 CLI integration tests for ONT-4c3 parity receipts pass, verifying pass/fail/decline discrimination over 5 fixtures and ensuring contract byte-identical parity.", + "command": "cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts", + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "grounding": "measured", + "line": 1, + "output": "test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 7.97s" + }, + { + "claim": "The independent denominator predicate script passes its 4 self-test cases and confirms 7 receipts under evidence/parity/** match EXPECTED_RECEIPTS.", + "command": "bash scripts/parity_receipt_denominator.sh --self-test && bash scripts/parity_receipt_denominator.sh", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 1, + "output": "ok a record is counted and an unrelated document is not (measured 1)\nok an unmigrated legacy record is refused by name\nok a receipt added without bumping the denominator disagrees\nok bumping the denominator makes them agree\nPASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." + }, + { + "claim": "Defines the active parity-receipt-v2 contract with 3 closed/specific shapes (parity-receipt-complete, parity-comparator-self, parity-comparator-oracle) and pins extractor reach via EXPECTED_RECEIPTS.", + "file": "contracts/parity-receipt-v2.yaml", + "grounding": "cited", + "line": 555 + } + ], + "raw_bytes": 10265, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "contracts/parity-receipt-v1.yaml", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The diff correctly leaves parity-receipt-v1 shapeless in accordance with the amendment of PMAT-3577." + ] + }, + { + "file": "contracts/parity-receipt-v1.yaml", + "line": 31, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The v1 receipt layout declares no shape, strictly abiding by the STOP condition 'do NOT add a v1 shape' as described in the amendment." + ] + }, + { + "file": "contracts/parity-receipt-v2.yaml", + "line": 55, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The shape dynamically resolves the threshold via thresholdSource using 'resolves: path' instead of hardcoding any threshold value, satisfying the STOP condition." + ] + }, + { + "file": "contracts/parity-receipt-v2.yaml", + "line": 555, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Defines the active parity-receipt-v2 contract with 3 closed/specific shapes (parity-receipt-complete, parity-comparator-self, parity-comparator-oracle) and pins extractor reach via EXPECTED_RECEIPTS." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The tests for ont4c3 parity receipts pass, including the negative condition for a comparator kind the shape does not accept." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "All 9 CLI integration tests for ONT-4c3 parity receipts pass, verifying pass/fail/decline discrimination over 5 fixtures and ensuring contract byte-identical parity." + ] + }, + { + "file": "crates/aprender-contracts/src/lint/shapes_gate.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The gate shapes_gate was not weakened; it was actually strengthened to return WrongCorpus on a mismatch." + ] + }, + { + "file": "crates/aprender-contracts/src/ontology/extract/parity_receipt_tests.rs", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "extract:parity-receipt unit tests pass cleanly with 10 passed tests, verifying classification, legacy record refusal by name, missing threshold detection, positive control, and denominator checks." + ] + }, + { + "file": "evidence/parity/EXPECTED_RECEIPTS", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The denominator predicate correctly expects and matches 7 receipts." + ] + }, + { + "file": "evidence/parity/EXPECTED_RECEIPTS", + "line": 9, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The denominator count for EXPECTED_RECEIPTS is correctly pinned to 7, exactly matching the number of back-filled receipts." + ] + }, + { + "file": "scripts/check_parity_receipt.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The check_parity_receipt.sh script was left untouched." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The independent denominator predicate script passes its 4 self-test cases and confirms 7 receipts under evidence/parity/** match EXPECTED_RECEIPTS." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (10265 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)" + ], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3601,PMAT-3656.json b/docs/audits/quorum-PMAT-3601,PMAT-3656.json new file mode 100644 index 0000000000..a37ddedb8a --- /dev/null +++ b/docs/audits/quorum-PMAT-3601,PMAT-3656.json @@ -0,0 +1,231 @@ +{ + "ticket": "PMAT-3601,PMAT-3656", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "171a0b2938439d9b8c385a3fe9ea249e19d16dfd", + "diff_sha256": "87f49f1c8079bdf2048475e6320bf713ce5397393fc2196e6857525f124e1c2d", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 50043, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff meticulously implements all requirements for both PMAT-3601 and PMAT-3656. All ad-hoc size/name tables have been removed in favor of delegating to `trueno_quant::TRAITS`, ensuring types are perfectly sized and named. A comprehensive suite of regression, backward-compatibility, and error-handling tests was introduced, strictly asserting that unknown IDs or missing dequantizers produce well-formatted, contract-referencing errors rather than inventing approximate weights or raising confusing format errors. Contracts and roadmaps were also correspondingly updated. There is no evidence of regression or unsupported behavior.", + "findings": [ + { + "claim": "apr inspect succeeds on a GGUF containing any of the 10 listed ids (repro: Qwen3.5-4B-UD-Q4_K_XL.gguf, dtype 23)", + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "fix": "Added `test_3601_rosetta_inspect_accepts_an_iq4_xs_gguf` which explicitly builds a synthetic GGUF with dtype 23 (IQ4_XS) and ensures `inspect()` extracts the name and size correctly.", + "grounding": "cited", + "line": 181 + }, + { + "claim": "A regression test pins all 10 ids against shape.rs's byte_size", + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "fix": "Added `test_3601_get_tensor_raw_sizes_every_iq_and_tq_type` which tests 11 IDs (including 16..=23) against `get_tensor_raw()` to verify exactly correct byte sizing in bytes/blocks.", + "grounding": "cited", + "line": 47 + }, + { + "claim": "The error message, if a truly unknown id is ever hit, names the id and points at contracts/apr-cli-commands-v1.yaml's inspect docs or similar — not a bare \"Unsupported dtype N\"", + "file": "crates/aprender-core/src/format/gguf/reader.rs", + "fix": "`unsized_ggml_type_error` now names the tensor and type ID, explains why it failed, and points to `contracts/ggml-type-v1.yaml`. A test `test_3601_unknown_id_is_refused_as_a_gap_in_apr` asserts this format.", + "grounding": "cited", + "line": 257 + }, + { + "claim": "apr tensors sized tensors from a second hand-typed table ... now type_size/blck_size from TRAITS", + "file": "crates/aprender-core/src/format/safetensors.rs", + "fix": "`ggml_dtype_element_size` was rewritten to calculate size via `trueno_quant::GgmlType::from_id`. `tensors_tests_stats.rs` was updated to verify the corrected byte elements for IQ-types and the 24-30 tail.", + "grounding": "cited", + "line": 10 + }, + { + "claim": "ggml_dtype_name ... reads TRAITS so TQ1_0/TQ2_0 are named rather than \"unknown\"", + "file": "crates/aprender-core/src/format/tensors.rs", + "fix": "`ggml_dtype_name` delegates to `trueno_quant::GgmlType::from_id`. `test_3601_ggml_dtype_name_keeps_0_to_30_and_names_the_ids_past_it` checks backward compatibility for 0..=30 and correct naming for IDs like TQ1_0.", + "grounding": "cited", + "line": 572 + }, + { + "claim": "get_tensor_f32 on any IQ tensor returns Err naming the tensor and its type; no path in aprender-core returns approximated IQ values; dequantize_iq_approximate is deleted", + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "fix": "`dequantize_iq_approximate` was completely removed from `dequantize.rs`. The match arm for IQ types in `shape.rs` now falls through to an explicit refusal error naming the tensor and its ID. This is backed by `test_3656_get_tensor_f32_refuses_iq_and_tq_instead_of_approximating`.", + "grounding": "cited", + "line": 131 + } + ], + "raw_bytes": 8820, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully resolves both tickets PMAT-3601 and PMAT-3656. It transitions `aprender-core` from using hardcoded, incomplete tables to reading `trueno_quant::TRAITS` for tensor block sizes and names. It also removes the `dequantize_iq_approximate` function to prevent inventing IQ-quant weights during import/conversion, and returns clear, descriptive error messages for unsupported or unknown types, pointing to relevant contracts. The regression tests fully cover the new functionality.", + "findings": [ + { + "claim": "apr inspect sizes and names ggml types from trueno_quant::TRAITS, removing the hardcoded tables for sizing and naming.", + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "grounding": "cited" + }, + { + "claim": "Removes the IQ approximation in dequantization and properly refuses the formats it does not have a dequantizer for.", + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "grounding": "cited" + }, + { + "claim": "The error message for a truly unknown id names the id (via Display on GgmlTypeError) and points at contracts/ggml-type-v1.yaml, avoiding a bare 'Unsupported dtype N'.", + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "grounding": "cited" + }, + { + "claim": "Regression test pins all 10 ids against shape.rs's byte_size.", + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "grounding": "cited" + } + ], + "raw_bytes": 4837, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed diff origin/main...HEAD against PMAT-3601 and PMAT-3656 tickets. The diff completely resolves both tickets by replacing hand-typed GGML size and name tables with trueno_quant::TRAITS in aprender-core (shape.rs, safetensors.rs, tensors.rs), deleting the IQ approximation code (dequantize_iq_approximate), and enforcing explicit refusal on get_tensor_f32 for unsupported IQ/TQ types while correctly sizing raw tensors for apr inspect / apr tensors. Contract invariants, Kani bounds, and comprehensive unit tests were updated and added. All 3,979 unit tests pass cleanly. No gates weakened, no unasked changes, no refutations found. Verdict: PASS.", + "findings": [], + "raw_bytes": 3584, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-core/src/format/gguf/reader.rs", + "line": 257, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The error message, if a truly unknown id is ever hit, names the id and points at contracts/apr-cli-commands-v1.yaml's inspect docs or similar — not a bare \"Unsupported dtype N\"" + ] + }, + { + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Regression test pins all 10 ids against shape.rs's byte_size." + ] + }, + { + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "line": 47, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "A regression test pins all 10 ids against shape.rs's byte_size" + ] + }, + { + "file": "crates/aprender-core/src/format/gguf/reader_tests_ggml_sizes.rs", + "line": 181, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "apr inspect succeeds on a GGUF containing any of the 10 listed ids (repro: Qwen3.5-4B-UD-Q4_K_XL.gguf, dtype 23)" + ] + }, + { + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Removes the IQ approximation in dequantization and properly refuses the formats it does not have a dequantizer for.", + "The error message for a truly unknown id names the id (via Display on GgmlTypeError) and points at contracts/ggml-type-v1.yaml, avoiding a bare 'Unsupported dtype N'.", + "apr inspect sizes and names ggml types from trueno_quant::TRAITS, removing the hardcoded tables for sizing and naming." + ] + }, + { + "file": "crates/aprender-core/src/format/gguf/shape.rs", + "line": 131, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "get_tensor_f32 on any IQ tensor returns Err naming the tensor and its type; no path in aprender-core returns approximated IQ values; dequantize_iq_approximate is deleted" + ] + }, + { + "file": "crates/aprender-core/src/format/safetensors.rs", + "line": 10, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "apr tensors sized tensors from a second hand-typed table ... now type_size/blck_size from TRAITS" + ] + }, + { + "file": "crates/aprender-core/src/format/tensors.rs", + "line": 572, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "ggml_dtype_name ... reads TRAITS so TQ1_0/TQ2_0 are named rather than \"unknown\"" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3602.json b/docs/audits/quorum-PMAT-3602.json new file mode 100644 index 0000000000..f689ad9116 --- /dev/null +++ b/docs/audits/quorum-PMAT-3602.json @@ -0,0 +1,248 @@ +{ + "ticket": "PMAT-3602", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "f1810100d84df45f650ea5c889b929884586fafe", + "diff_sha256": "811dccb839cd0eecd4f94a3ef705b5ad4df1b421b2968cf065b95564e004b4d4", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 41550, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements PMAT-3602, surfacing GPU rejection explicitly without leaking it as a success. `reconcile_accelerator` ensures `registry::after_generation` correctly processes a fallback by throwing exit 14 (CliError::BackendUnavailable) for requested GPUs. The machine output (JSON/Stream) now clearly includes `fell_back: true` distinguishing it from intentional CPU runs, while human output correctly abstains from misleading success messages. The tests effectively guard against regressions.", + "findings": [ + { + "claim": "reconcile_and_emit correctly wires registry::after_generation for forced GPU checks. It returns the refusal (CliError::BackendUnavailable) effectively surfacing the error to stderr and emitting exit 14.", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 146 + }, + { + "claim": "build_final_json reliably adds the `backend` key structure explicitly isolating an unintentional GPU fallback (fell_back: true) from an intentional CPU run, satisfying the requirement to surface GPU rejection on machine output surfaces (--json/--stream).", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 527 + }, + { + "claim": "The diff correctly avoids misclassification by uniformly leveraging `registry::Request::wanted()`. It accurately assumes `layers_want_accelerator: false` since `--gpu-layers` belongs exclusively to `apr serve`.", + "file": "crates/apr-cli/src/dispatch.rs", + "grounding": "cited", + "line": 207 + }, + { + "claim": "Six thorough unit tests verify that forced accelerator fallbacks return an error while silent intentional CPU runs behave correctly, satisfying Item 5's test coverage requirement.", + "file": "crates/apr-cli/src/commands/run_tests_accel_reconcile.rs", + "grounding": "cited", + "line": 56 + } + ], + "raw_bytes": 5689, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements the requirements for PMAT-3602. It records the fail-vs-fall-back decision by wiring `registry::after_generation` into `apr run`, surfacing GPU rejections on machine surfaces (`--json`) while preventing human success output on fallbacks. It accurately adds the 6 required falsification tests, proving the fix works without breaking deliberate CPU runs. The change appropriately ignores the `--gpu-layers` flag for `apr run` and avoids approximating the un-plumbed rejection reason, adhering to the ticket's negative constraints.", + "findings": [ + { + "claim": "The diff successfully wires `registry::after_generation` into the production execution path inside `reconcile_accelerator` for `apr run`. It explicitly throws an error when `accel_forced` is true but the model executed on CPU, converting the silent fallback into a refusal.", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 146 + }, + { + "claim": "The json output has been augmented with a `backend` object (`requested`, `ran`, `fell_back`) which accurately records the engine's execution mode, preventing a fallback from being indistinguishable from a deliberate CPU run.", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 542 + }, + { + "claim": "The test file `run_tests_accel_reconcile.rs` was created and includes the required 6 cases to ensure `reconcile_accelerator` properly refuses forced CPU fallbacks while remaining silent on deliberate CPU runs and healthy GPU runs.", + "file": "crates/apr-cli/src/commands/run_tests_accel_reconcile.rs", + "grounding": "cited", + "line": 1 + }, + { + "claim": "Accelerator intent classification was consolidated in `dispatch.rs` using `crate::registry::Request::wanted()` so that CLI dispatch shares the same logic as the registry.", + "file": "crates/apr-cli/src/dispatch.rs", + "grounding": "cited", + "line": 208 + } + ], + "raw_bytes": 6057, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed diff origin/main...HEAD against ticket PMAT-3602. The diff successfully wires `registry::after_generation` into the production execution path in `run_entry.rs`, ensuring `apr run --gpu` returns exit 14 (`CliError::BackendUnavailable`) on CPU fallback instead of reporting false success. Machine surfaces (`--json` and `--stream`) properly surface the fallback via `\"backend\": {\"requested\": \"gpu\", \"ran\": \"cpu\", \"fell_back\": true}` prior to returning the refusal error, while human surfaces remain silent. Falsification unit tests cover all cases (forced GPU run on CPU, forced GPU run on GPU, unforced CPU run, unforced GPU run, unreporting backend, and JSON output formatting). No refutations found.", + "findings": [ + { + "claim": "reconcile_accelerator invokes registry::after_generation(true, Some(\"gpu\"), result.used_gpu) when accel_forced is true, returning CliError::BackendUnavailable when used_gpu is Some(false)", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 207 + }, + { + "claim": "emits_machine_output ensures machine formats (--json and --stream) surface backend status while skipping human success output on forced accelerator rejection", + "file": "crates/apr-cli/src/commands/run_entry.rs", + "grounding": "cited", + "line": 188 + }, + { + "claim": "dispatch.rs classifies accelerator requests via registry::Request::wanted() ensuring uniform classification between CLI dispatch and reconciliation", + "file": "crates/apr-cli/src/dispatch.rs", + "grounding": "cited", + "line": 207 + } + ], + "raw_bytes": 5247, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands/run_entry.rs", + "line": 146, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "The diff successfully wires `registry::after_generation` into the production execution path inside `reconcile_accelerator` for `apr run`. It explicitly throws an error when `accel_forced` is true but the model executed on CPU, converting the silent fallback into a refusal.", + "reconcile_and_emit correctly wires registry::after_generation for forced GPU checks. It returns the refusal (CliError::BackendUnavailable) effectively surfacing the error to stderr and emitting exit 14." + ] + }, + { + "file": "crates/apr-cli/src/commands/run_entry.rs", + "line": 188, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "emits_machine_output ensures machine formats (--json and --stream) surface backend status while skipping human success output on forced accelerator rejection" + ] + }, + { + "file": "crates/apr-cli/src/commands/run_entry.rs", + "line": 207, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "reconcile_accelerator invokes registry::after_generation(true, Some(\"gpu\"), result.used_gpu) when accel_forced is true, returning CliError::BackendUnavailable when used_gpu is Some(false)" + ] + }, + { + "file": "crates/apr-cli/src/commands/run_entry.rs", + "line": 527, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "build_final_json reliably adds the `backend` key structure explicitly isolating an unintentional GPU fallback (fell_back: true) from an intentional CPU run, satisfying the requirement to surface GPU rejection on machine output surfaces (--json/--stream)." + ] + }, + { + "file": "crates/apr-cli/src/commands/run_entry.rs", + "line": 542, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The json output has been augmented with a `backend` object (`requested`, `ran`, `fell_back`) which accurately records the engine's execution mode, preventing a fallback from being indistinguishable from a deliberate CPU run." + ] + }, + { + "file": "crates/apr-cli/src/commands/run_tests_accel_reconcile.rs", + "line": 1, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The test file `run_tests_accel_reconcile.rs` was created and includes the required 6 cases to ensure `reconcile_accelerator` properly refuses forced CPU fallbacks while remaining silent on deliberate CPU runs and healthy GPU runs." + ] + }, + { + "file": "crates/apr-cli/src/commands/run_tests_accel_reconcile.rs", + "line": 56, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "Six thorough unit tests verify that forced accelerator fallbacks return an error while silent intentional CPU runs behave correctly, satisfying Item 5's test coverage requirement." + ] + }, + { + "file": "crates/apr-cli/src/dispatch.rs", + "line": 207, + "lanes_agreeing": [ + 1, + 3 + ], + "claims": [ + "The diff correctly avoids misclassification by uniformly leveraging `registry::Request::wanted()`. It accurately assumes `layers_want_accelerator: false` since `--gpu-layers` belongs exclusively to `apr serve`.", + "dispatch.rs classifies accelerator requests via registry::Request::wanted() ensuring uniform classification between CLI dispatch and reconciliation" + ] + }, + { + "file": "crates/apr-cli/src/dispatch.rs", + "line": 208, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Accelerator intent classification was consolidated in `dispatch.rs` using `crate::registry::Request::wanted()` so that CLI dispatch shares the same logic as the registry." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3604.json b/docs/audits/quorum-PMAT-3604.json new file mode 100644 index 0000000000..e31fdd6507 --- /dev/null +++ b/docs/audits/quorum-PMAT-3604.json @@ -0,0 +1,169 @@ +{ + "ticket": "PMAT-3604", + "base": "main", + "base_resolved": "origin/main", + "base_note": "local main differs from origin/main by 128 commit(s); judged against origin/main", + "head": "dd546592b0d47863877f43fac52246120b392ed8", + "diff_sha256": "092ba20f3667a8be79d39da6d26b687eababcfa66423fa7ff1ba1a43404f23fd", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 49882, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly satisfies the requirements of PMAT-3604. It introduces receipting for the F2 hybrid guard based on the (model sha256, apr version, device) triple and adds comprehensive unit testing to ensure no cache collisions or improper validations can occur. Edge cases where the guard fails to evaluate real tokens are properly handled with a 'NotJudged' verdict, bypassing the gate safely without storing an invalid receipt that could weaken the gate for future calls.", + "findings": [ + { + "claim": "The diff correctly keys the receipt on model_sha256, apr_version, and device, validating and matching them exactly before skipping the guard forward.", + "file": "crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs", + "grounding": "cited", + "line": 144 + }, + { + "claim": "The F2 guard explicitly returns a NotJudged verdict when evaluation is skipped (e.g. prompt length < 2 or SKIP_PARITY_GATE=1). This correctly prevents such calls from writing a receipt, ensuring the gate is not permanently bypassed.", + "file": "crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs", + "grounding": "cited", + "line": 1515 + }, + { + "claim": "The implementation safely sets the APR_F2_REVALIDATE environment variable to support the --revalidate flag before any background thread starts.", + "file": "crates/apr-cli/src/dispatch.rs", + "grounding": "cited", + "line": 149 + } + ], + "raw_bytes": 4867, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements the requested receipt mechanism for the F2 CPU parity guard. It checks the model sha256, apr version, and device name, and appropriately skips the CPU forward when a valid receipt exists. The tests thoroughly verify the exact decision table requirements. No claims or assertions are refuted, although there is a slight performance overhead introduced for skipped gates.", + "findings": [ + { + "claim": "The diff adds a --revalidate CLI flag to force a fresh validation, which is a feature not explicitly requested in the ticket PMAT-3604.", + "file": "crates/apr-cli/src/commands_enum.rs", + "fix": "Remove the --revalidate flag and rely only on file deletion to revalidate, or accept it as a reasonable testing addition.", + "grounding": "cited", + "line": 139 + }, + { + "claim": "The whole-file SHA-256 is computed before checking the `SKIP_PARITY_GATE` environment variable. This adds a ~1s hash penalty to every run even when the user explicitly set `SKIP_PARITY_GATE=1` to bypass the guard entirely.", + "file": "crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs", + "fix": "Check for `SKIP_PARITY_GATE=1` in `f2_validate_qwen35_receipted` and return early before hashing the model.", + "grounding": "cited", + "line": 1558 + } + ], + "raw_bytes": 4603, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Reviewed PMAT-3604 diff against all ticket requirements. The implementation successfully caches and receipts the F2 CPU/GPU parity guard per (model sha256, apr version, device) triple, reducing TTFT from ~17.18s to ~7.39s (saving ~9.8s). All falsification tests (planted receipts with wrong model sha256, wrong apr version, wrong device, corrupted file, missing receipt, --revalidate override) pass cleanly and fail closed as expected. Gates are strictly preserved and unjudged/failed runs never leave behind a false receipt. No refuting findings found.", + "findings": [], + "raw_bytes": 5406, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands_enum.rs", + "line": 139, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The diff adds a --revalidate CLI flag to force a fresh validation, which is a feature not explicitly requested in the ticket PMAT-3604." + ] + }, + { + "file": "crates/apr-cli/src/dispatch.rs", + "line": 149, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The implementation safely sets the APR_F2_REVALIDATE environment variable to support the --revalidate flag before any background thread starts." + ] + }, + { + "file": "crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs", + "line": 144, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The diff correctly keys the receipt on model_sha256, apr_version, and device, validating and matching them exactly before skipping the guard forward." + ] + }, + { + "file": "crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs", + "line": 1515, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The F2 guard explicitly returns a NotJudged verdict when evaluation is skipped (e.g. prompt length < 2 or SKIP_PARITY_GATE=1). This correctly prevents such calls from writing a receipt, ensuring the gate is not permanently bypassed." + ] + }, + { + "file": "crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs", + "line": 1558, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The whole-file SHA-256 is computed before checking the `SKIP_PARITY_GATE` environment variable. This adds a ~1s hash penalty to every run even when the user explicitly set `SKIP_PARITY_GATE=1` to bypass the guard entirely." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3605.json b/docs/audits/quorum-PMAT-3605.json new file mode 100644 index 0000000000..e3ea20aeb4 --- /dev/null +++ b/docs/audits/quorum-PMAT-3605.json @@ -0,0 +1,201 @@ +{ + "ticket": "PMAT-3605", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "0cd2c6aee7d38b8024c97b8bf46f9a30addddb7a", + "diff_sha256": "0786058175344b37587bbf439231e7c79b53085bba429a2468ec83c190a15eca", + "width": 3, + "executor": "agy", + "prompt_mode": "file", + "prompt_bytes": 125886, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements PMAT-3605. It creates the refusal-receipt-v1 shape with `removed_by` configured as a mandatory field (minCount: 1) matching the exact closed set pattern `^(v[0-9]+\\.[0-9]+|never|unscheduled)$`. It includes fixtures for each accepted sentinel arm as well as rejecting edge cases (such as 'tbd', sha, and bare versions). Both the ont_refusal_receipt and ont4b_shapes_gate tests pass successfully. There are no violations of the ticket's negative criteria.", + "findings": [ + { + "claim": "The ont_refusal_receipt test suite successfully verifies the requirements, passing 9/9 tests including all arms of the alternation and rejection cases.", + "command": "cargo test -p aprender-contracts-cli --test ont_refusal_receipt", + "file": "crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs", + "grounding": "measured", + "line": 442, + "output": "test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s" + }, + { + "claim": "The ont4b_shapes_gate test suite passes 11/11 tests, successfully verifying the tracked repo graph is fresh and confirming shapes_n equals 6.", + "command": "cargo test -p aprender-contracts-cli --test ont4b_shapes_gate", + "file": "crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs", + "grounding": "measured", + "line": 228, + "output": "test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 13.86s" + }, + { + "claim": "refusal-receipt-v1.yaml defines removed_by as a required property (minCount: 1) and enforces a closed pattern ^(v[0-9]+\\.[0-9]+|never|unscheduled)$, satisfying the ticket criteria.", + "file": "contracts/refusal-receipt-v1.yaml", + "grounding": "cited", + "line": 226 + } + ], + "raw_bytes": 5709, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The provided diff correctly implements PMAT-3605. The 'removed_by' field is constrained by a shape in 'refusal-receipt-v1.yaml' using a closed set pattern ('^(v[0-9]+\\\\.[0-9]+|never|unscheduled)$') and is mandatory (minCount: 1). The test fixtures cover the acceptable arms and rejection cases.", + "findings": [ + { + "claim": "The diff implements a contract 'refusal-receipt-v1.yaml' with a shape over 'removed_by' that uses a closed set pattern for its values, ensuring it is required and not free text.", + "file": "contracts/refusal-receipt-v1.yaml", + "grounding": "cited" + } + ], + "raw_bytes": 3256, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Diff for ticket PMAT-3605 (#3605) completely and correctly implements 'removed_by gets a shape: refusal-receipt-v1 with a closed escape set'. The new pattern contract `contracts/refusal-receipt-v1.yaml` defines a closed set regex pattern `^(v[0-9]+\\\\.[0-9]+|never|unscheduled)$` with `minCount: 1` on `removed_by`, preventing free text and fabricated releases. Eight fixture test cases in `tests/fixtures/ont/` and `crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs` test both valid sentinels (v0.70, never, unscheduled) and invalid entries (tbd, sha, bare version, patch version, missing field, terse reason). `shapes_n` count in `ont4b_shapes_gate.rs` was updated from 5 to 6, and all tests pass without drift.", + "findings": [ + { + "claim": "The ont_refusal_receipt test suite passes 9/9 tests, verifying both accepted sentinels (v0.70, never, unscheduled) and rejected edge cases (tbd, sha, bare version 0.70, patch version v0.70.1, missing removed_by, no exit_code, terse reason).", + "command": "cargo test -p aprender-contracts-cli --test ont_refusal_receipt", + "file": "crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs", + "grounding": "measured", + "line": 442, + "output": "test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s" + }, + { + "claim": "The ont4b_shapes_gate test suite passes 11/11 tests, confirming the tracked repo graph assertion for shapes_n == 6.", + "command": "cargo test -p aprender-contracts-cli --test ont4b_shapes_gate", + "file": "crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs", + "grounding": "measured", + "line": 228, + "output": "test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 16.42s" + }, + { + "claim": "pv extract contracts --check verifies zero drift across 15620 triples and 6 shapes.", + "command": "cargo run -p aprender-contracts-cli --bin pv -- extract contracts --check", + "file": "contracts/census.json", + "grounding": "measured", + "line": 32, + "output": "{\n \"triples\": 15620,\n \"sha256\": \"54e4a15ab7f6e4415aa7959a51826f731b650a1998e80b635e6850ae8f5df283\",\n \"shapes_n\": 6,\n \"written\": [],\n \"check\": []\n}" + }, + { + "claim": "contracts/refusal-receipt-v1.yaml defines removed_by as a required property (minCount: 1) with a closed pattern pattern: ^(v[0-9]+\\.[0-9]+|never|unscheduled)$.", + "file": "contracts/refusal-receipt-v1.yaml", + "grounding": "cited", + "line": 226 + } + ], + "raw_bytes": 7209, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "contracts/census.json", + "line": 32, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "pv extract contracts --check verifies zero drift across 15620 triples and 6 shapes." + ] + }, + { + "file": "contracts/refusal-receipt-v1.yaml", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The diff implements a contract 'refusal-receipt-v1.yaml' with a shape over 'removed_by' that uses a closed set pattern for its values, ensuring it is required and not free text." + ] + }, + { + "file": "contracts/refusal-receipt-v1.yaml", + "line": 226, + "lanes_agreeing": [ + 1, + 3 + ], + "claims": [ + "contracts/refusal-receipt-v1.yaml defines removed_by as a required property (minCount: 1) with a closed pattern pattern: ^(v[0-9]+\\.[0-9]+|never|unscheduled)$.", + "refusal-receipt-v1.yaml defines removed_by as a required property (minCount: 1) and enforces a closed pattern ^(v[0-9]+\\.[0-9]+|never|unscheduled)$, satisfying the ticket criteria." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont4b_shapes_gate.rs", + "line": 228, + "lanes_agreeing": [ + 1, + 3 + ], + "claims": [ + "The ont4b_shapes_gate test suite passes 11/11 tests, confirming the tracked repo graph assertion for shapes_n == 6.", + "The ont4b_shapes_gate test suite passes 11/11 tests, successfully verifying the tracked repo graph is fresh and confirming shapes_n equals 6." + ] + }, + { + "file": "crates/aprender-contracts-cli/tests/ont_refusal_receipt.rs", + "line": 442, + "lanes_agreeing": [ + 1, + 3 + ], + "claims": [ + "The ont_refusal_receipt test suite passes 9/9 tests, verifying both accepted sentinels (v0.70, never, unscheduled) and rejected edge cases (tbd, sha, bare version 0.70, patch version v0.70.1, missing removed_by, no exit_code, terse reason).", + "The ont_refusal_receipt test suite successfully verifies the requirements, passing 9/9 tests including all arms of the alternation and rejection cases." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3618.json b/docs/audits/quorum-PMAT-3618.json new file mode 100644 index 0000000000..4c92b3f17f --- /dev/null +++ b/docs/audits/quorum-PMAT-3618.json @@ -0,0 +1,241 @@ +{ + "ticket": "PMAT-3618", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "e9ca589997b042bb38572975c812fda8806be123", + "diff_sha256": "333c8651632d2988647554afdab71cb73b33e53d41c247688afaa37b64840450", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 38781, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff cleanly resolves PMAT-3618 without weakening any gates or making unauthorized changes. \n\n1. **AC 1 & 2 Satisfied:** All literal release identities (`V`, `T`, `MS`, `EPIC`, `LAST_TAG`) and the hardcoded `AP=/mnt/...` path have been replaced with derivations powered by a new `lib_release_params.sh` script. `AP` now properly resolves to the main repository checkout (`target/release-train/$T`).\n2. **AC 3 Satisfied:** The diff ships its own robust assertion script (`check_release_scripts_derive_identity.sh`). It scans all scripts under `scripts/release/` for paths matching `/mnt`, `/home`, etc. (R1) and literal identity assignments (R2). It properly skips comments for R2 and has strict anti-vacuity checks.\n3. **AC 4 Satisfied:** The assertion script includes a `--self-test` mechanism that perfectly models the falsifier requirement: it explicitly tests the `#3618` `/mnt/nvme-raid0/` shape and confirms that the script correctly flags it and fails.\n\nAll other modifications (such as adapting positional arguments in `publish_strict.sh` and `prepare_bump.sh` to accept the `` argument) are necessary consequences of the parameterization and perfectly execute the ticket's intent. No refutations found.", + "findings": [ + { + "claim": "The literal release identities (V, T, MS, EPIC, LAST_TAG) and out-of-repo AP path have been removed from autopilot.sh and other release scripts, satisfying AC 1 and AC 2. They are now dynamically resolved via lib_release_params.sh", + "file": "scripts/release/autopilot.sh", + "grounding": "cited", + "line": 14 + }, + { + "claim": "The row ships its own assertion gate via check_release_scripts_derive_identity.sh to forbid hardcoded literals (R2_RE) and out-of-repo paths (R1_RE), satisfying AC 3.", + "file": "scripts/check_release_scripts_derive_identity.sh", + "grounding": "cited", + "line": 29 + }, + { + "claim": "The assertion gate includes a self-test falsifier that explicitly verifies the gate goes RED when a literal `/mnt/nvme-raid0/...` path is reintroduced, satisfying AC 4.", + "file": "scripts/check_release_scripts_derive_identity.sh", + "grounding": "cited", + "line": 67 + } + ], + "raw_bytes": 6382, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements the requested changes: V/T/MS/EPIC and AP are dynamically derived rather than hardcoded, and out-of-repo paths (like /mnt/) are removed. A new dedicated assertion script `scripts/check_release_scripts_derive_identity.sh` is introduced to enforce this constraint continuously, meeting the ticket's explicit branch 2 requirement. Testing confirms the script correctly flags literal paths.", + "findings": [ + { + "claim": "V/T/MS/EPIC and AP are derived, not literals", + "file": "scripts/release/autopilot.sh", + "grounding": "cited" + }, + { + "claim": "Shipped assertion to enforce no out-of-repo paths or literal identities", + "file": "scripts/check_release_scripts_derive_identity.sh", + "grounding": "cited" + }, + { + "claim": "Falsifier gate accurately goes RED when tested against an out-of-repo path", + "file": "scripts/check_release_scripts_derive_identity.sh", + "grounding": "cited", + "grounding_downgraded": "measured-without-command-or-output" + } + ], + "raw_bytes": 3908, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "As Lane 3 reviewer, I evaluated the diff against ticket PMAT-3618 and verified all acceptance criteria.\n\nKey Findings & Measured Verifications:\n1. Identity Derivation: `scripts/release/lib_release_params.sh` dynamically derives release identity parameters (`V`, `T`, `AP`, `MS`, `EPIC`, `LAST_TAG`). Scripts (`autopilot.sh`, `prepare_bump.sh`, `publish_strict.sh`, `t2_preflight.sh`, `ledger_from_run.sh`) no longer hardcode 0.68.2 constants or `/mnt/...` out-of-repo paths.\n2. In-Tree Assertion Guard: Introduced `scripts/check_release_scripts_derive_identity.sh` which enforces R1 (no out-of-repo paths) and R2 (no literal identity assignments) across `scripts/release/`. Its self-test suite passes 14 case table rows.\n3. Falsifier Verified: Reintroducing a literal `AP=/mnt/nvme-raid0/test` path into `scripts/release/autopilot.sh` causes `check_release_scripts_derive_identity.sh` to go RED (exit code 1).\n4. CI Wiring Verified: `guard_tree.sh` automatically discovers `scripts/check_release_scripts_derive_identity.sh` via globbing, ensuring execution on every PR/commit in CI.\n5. Preflight Self-Test Verified: `t2_preflight.sh --self-test` passes all 12 case table checks.\n\nNo refuting issues, weakened gates, unbacked receipt claims, or scope violations were found. Verdict: PASS.", + "findings": [ + { + "claim": "scripts/check_release_scripts_derive_identity.sh passes self-test suite (14 case table rows) and validates scripts/release/ cleanroom", + "command": "bash scripts/check_release_scripts_derive_identity.sh --self-test && bash scripts/check_release_scripts_derive_identity.sh", + "file": "scripts/check_release_scripts_derive_identity.sh", + "fix": "N/A", + "grounding": "measured", + "line": 1, + "output": "=== release scripts derive their identity: case table ===\nok row 1 rc=0 derivations only -> clean\nok row 2 rc=1 the #3618 shape: AP under the RAID mount -> RED (R1 and R2)\nok row 3 rc=1 that path in a COMMENT -> RED (R1 reads comments)\nok row 4 rc=1 a home-directory path -> RED\nok row 5 rc=1 V=0.69.0 -> RED (R2)\nok row 6 rc=1 MS and EPIC literals after another statement -> RED\nok row 7 rc=1 LAST_TAG=v0.68.1 -> RED\nok row 8 rc=1 a quoted literal V=\"0.69.0\" -> RED\nok row 9 rc=0 an assignment in a COMMENT is documentation -> clean (R2 skips comments)\nok row 10 rc=0 lowercase locals and names that merely end in V/T -> clean\nok row 11 rc=0 /dev/null, /tmp and $HOME paths are not operator-box literals -> clean\nok row 12 rc=2 zero files -> ENV, never a pass\nok row 13 rc=2 missing directory -> ENV\nok row 14 rc=0 the real scripts/release is clean\nSELF-TEST PASSED: 14 rows\n=== scripts/release/ derives the release identity (check_release_scripts_derive_identity.sh) ===\nok 8 file(s) under scripts/release: no out-of-repo path, no literal release identity\nPASS" + }, + { + "claim": "Falsifier verification: reintroducing a literal /mnt/... path in scripts/release/autopilot.sh causes check_release_scripts_derive_identity.sh to fail with exit code 1 (RED)", + "command": "bash -c 'sed -i \"s/# Release autopilot/AP=\\/mnt\\/nvme-raid0\\/test\\n# Release autopilot/\" scripts/release/autopilot.sh; bash scripts/check_release_scripts_derive_identity.sh; rc=$?; git checkout scripts/release/autopilot.sh; exit $rc'", + "file": "scripts/release/autopilot.sh", + "fix": "N/A", + "grounding": "measured", + "line": 2, + "output": "=== scripts/release/ derives the release identity (check_release_scripts_derive_identity.sh) ===\nFAIL R1 scripts/release/autopilot.sh: a path outside the repository:\n 2:AP=/mnt/nvme-raid0/test\nFAIL R2 scripts/release/autopilot.sh: a release identity assigned a literal (derive it: lib_release_params.sh):\n 2:AP=/mnt/nvme-raid0/test\nFAIL (rc=1)" + }, + { + "claim": "Wiring verification: scripts/check_release_scripts_derive_identity.sh is automatically discovered and dispatched by guard_tree.sh in CI", + "command": "bash scripts/guard_tree.sh --dry-run | grep \"check_release_scripts_derive_identity\"", + "file": "scripts/check_release_scripts_derive_identity.sh", + "fix": "N/A", + "grounding": "measured", + "line": 1, + "output": "run: scripts/check_release_scripts_derive_identity.sh" + }, + { + "claim": "t2_preflight.sh self-test suite executes and passes 12 case table assertions", + "command": "bash scripts/release/t2_preflight.sh --self-test", + "file": "scripts/release/t2_preflight.sh", + "fix": "N/A", + "grounding": "measured", + "line": 56, + "output": "ok crash with zero rows is not GO\nok crash returns 2 (UNKNOWN)\nok missing log is not GO\nok missing log returns 2 (UNKNOWN)\nok no VERDICT line is not GO\nok truncated returns 2 (UNKNOWN)\nok row count below the declared floor is not GO\nok below-floor returns 2 (UNKNOWN)\nok complete run, only version-unpublished red -> GO\nok GO returns 0\nok a genuine FAIL row -> NO-GO\nok NO-GO returns 1\nself-test OK" + } + ], + "raw_bytes": 17442, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "scripts/check_release_scripts_derive_identity.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "Falsifier gate accurately goes RED when tested against an out-of-repo path", + "Shipped assertion to enforce no out-of-repo paths or literal identities" + ] + }, + { + "file": "scripts/check_release_scripts_derive_identity.sh", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Wiring verification: scripts/check_release_scripts_derive_identity.sh is automatically discovered and dispatched by guard_tree.sh in CI", + "scripts/check_release_scripts_derive_identity.sh passes self-test suite (14 case table rows) and validates scripts/release/ cleanroom" + ] + }, + { + "file": "scripts/check_release_scripts_derive_identity.sh", + "line": 29, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The row ships its own assertion gate via check_release_scripts_derive_identity.sh to forbid hardcoded literals (R2_RE) and out-of-repo paths (R1_RE), satisfying AC 3." + ] + }, + { + "file": "scripts/check_release_scripts_derive_identity.sh", + "line": 67, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The assertion gate includes a self-test falsifier that explicitly verifies the gate goes RED when a literal `/mnt/nvme-raid0/...` path is reintroduced, satisfying AC 4." + ] + }, + { + "file": "scripts/release/autopilot.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "V/T/MS/EPIC and AP are derived, not literals" + ] + }, + { + "file": "scripts/release/autopilot.sh", + "line": 2, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Falsifier verification: reintroducing a literal /mnt/... path in scripts/release/autopilot.sh causes check_release_scripts_derive_identity.sh to fail with exit code 1 (RED)" + ] + }, + { + "file": "scripts/release/autopilot.sh", + "line": 14, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The literal release identities (V, T, MS, EPIC, LAST_TAG) and out-of-repo AP path have been removed from autopilot.sh and other release scripts, satisfying AC 1 and AC 2. They are now dynamically resolved via lib_release_params.sh" + ] + }, + { + "file": "scripts/release/t2_preflight.sh", + "line": 56, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "t2_preflight.sh self-test suite executes and passes 12 case table assertions" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (17442 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)" + ], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3637.json b/docs/audits/quorum-PMAT-3637.json new file mode 100644 index 0000000000..90a8d14135 --- /dev/null +++ b/docs/audits/quorum-PMAT-3637.json @@ -0,0 +1,134 @@ +{ + "ticket": "PMAT-3637", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "bda16fc7b905c34139d41d2562a077db55bba10e", + "diff_sha256": "803cf607fa6fcd1aad51eb7c610b3ded0784efe5afb10da4caaa9ae79baab017", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 9184, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly adds PMAT-3604 and PMAT-3637 to the roadmap and entries. It is strictly additive in `roadmap.yaml` and does not touch any files outside of `docs/roadmaps/`. The criteria for PMAT-3637 are met.", + "findings": [ + { + "claim": "docs/roadmaps/entries/PMAT-3604.yaml is created with id PMAT-3604, github_issue 3604, kind:code and notes are present.", + "file": "docs/roadmaps/entries/PMAT-3604.yaml", + "grounding": "cited", + "line": 2 + }, + { + "claim": "docs/roadmaps/roadmap.yaml is modified additively with no lines deleted, adding PMAT-3604 and PMAT-3637.", + "file": "docs/roadmaps/roadmap.yaml", + "grounding": "cited", + "line": 6 + }, + { + "claim": "No files outside docs/roadmaps/ are modified in the provided diff.", + "file": "docs/roadmaps/roadmap.yaml", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 3669, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Review of PR #3637 (ticket PMAT-3637): PASS.\n\nThe diff strictly fulfills all acceptance criteria for PMAT-3637:\n1. `docs/roadmaps/entries/PMAT-3604.yaml` is added with id `PMAT-3604`, `github_issue: 3604`, `kind:code`, and accurately transcribes issue #3604's `done_when` criteria (items 1-6), admission rule, and out-of-scope list verbatim without omissions or unrequested additions.\n2. `docs/roadmaps/roadmap.yaml` equals `aggregate(entries/)` (verified via `scripts/lib/roadmap_fragments.py aggregate --check`) and the diff vs `origin/main` is strictly additive (`added=2, lifecycle=0, reserialised=0, deleted=0` verified via `scripts/check_roadmap_diff_additive.sh origin/main`).\n3. The PR body carries `keep-open: #3604`, and the diff touches only files inside `docs/roadmaps/` (`PMAT-3604.yaml`, `PMAT-3637.yaml`, and `roadmap.yaml`).\n4. All repository roadmap integrity gates (`check_roadmap_sorted.sh`, `check_roadmap_ids_unique.sh`, `check_roadmap_fragment_required.sh`, `check_roadmap_diff_additive.sh`) pass cleanly.\n5. No code, test, or unrelated files are touched outside the ticket's registration scope.", + "findings": [], + "raw_bytes": 4425, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.7-flash-high", + "model_measured": "gemini-3.7-flash-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 Audit Result: PASS. The diff correctly registers PMAT-3604 and PMAT-3637 in docs/roadmaps/entries/ and appends them additively to docs/roadmaps/roadmap.yaml. Done_when items 1-6 for issue #3604 are transcribed faithfully. The diff is strictly documentation/registration, touches nothing outside docs/roadmaps/, and contains no code or test changes for PMAT-3604 as specified.", + "findings": [], + "raw_bytes": 3031, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "docs/roadmaps/entries/PMAT-3604.yaml", + "line": 2, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "docs/roadmaps/entries/PMAT-3604.yaml is created with id PMAT-3604, github_issue 3604, kind:code and notes are present." + ] + }, + { + "file": "docs/roadmaps/roadmap.yaml", + "line": 1, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "No files outside docs/roadmaps/ are modified in the provided diff." + ] + }, + { + "file": "docs/roadmaps/roadmap.yaml", + "line": 6, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "docs/roadmaps/roadmap.yaml is modified additively with no lines deleted, adding PMAT-3604 and PMAT-3637." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3644.json b/docs/audits/quorum-PMAT-3644.json new file mode 100644 index 0000000000..6469a6c9ec --- /dev/null +++ b/docs/audits/quorum-PMAT-3644.json @@ -0,0 +1,293 @@ +{ + "ticket": "PMAT-3644", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "d18529e56a5d2199e69c21a3c7af04155ec22efa", + "diff_sha256": "97335d80da3ee08206561778de09e5acedc0b313570eb35d49009f69eba5a2b6", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 32102, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "All acceptance criteria have been verified via static diff analysis. The diff correctly drops `name:` lines before checking for dispatches, adds the 3 case rows testing the step name blindness, adjusts `check_pathonly_devdeps_unused_in_src.sh` to gracefully fallback or exit with ENV (rc=2) and implements the SCAN-DONE trailer, rewrites the build tool comments to remove it from `cargo-using`, expands `unwired_guards_baseline.txt` with the other 3 now-visible dark guards, and updates `lib_baseline_ratchet.sh` to handle NAME-entry admissions via `set-aperture` with exactly 6 case rows matching criteria. No workflow files were edited.", + "findings": [ + { + "claim": "check_guards_are_wired.sh drops name: lines before the invocation match, in dispatcher_wired() and the per-guard scan", + "file": "scripts/check_guards_are_wired.sh", + "grounding": "cited" + }, + { + "claim": "rows 8-10 check the exact ci.yml step-name fixture, a quoted run: line, and a step named after a guard wires nothing", + "file": "scripts/check_guards_are_wired.sh", + "grounding": "cited" + }, + { + "claim": "the mutation (remove the drop) turns rows 8 and 10 RED", + "file": "scripts/check_guards_are_wired.sh", + "grounding": "asserted" + }, + { + "claim": "check_pathonly_devdeps_unused_in_src.sh falls back to tomli, exits with ENV rc=2, and requires a SCAN-DONE trailer", + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "grounding": "cited" + }, + { + "claim": "five death rows incl. the whole guard under the intel shape", + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "grounding": "cited" + }, + { + "claim": "comments reworded so CARGO_RE no longer classifies it out of --no-cargo", + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "grounding": "cited" + }, + { + "claim": "unwired_guards_baseline.txt records the other three dark guards as APERTURE REVEALS", + "file": "scripts/unwired_guards_baseline.txt", + "grounding": "cited" + }, + { + "claim": "set-aperture gains a NAME-entry admission with six case rows for acceptance vs refusal", + "file": "scripts/lib_baseline_ratchet.sh", + "grounding": "cited" + }, + { + "claim": "No workflow file is edited", + "file": "docs/roadmaps/entries/PMAT-3644.yaml", + "grounding": "asserted" + } + ], + "raw_bytes": 7984, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly matches the requirements in PMAT-3644. All expected behaviors, tests, and file modifications requested in the acceptance criteria have been found exactly as specified.", + "findings": [ + { + "claim": "The name: dropping logic is wired correctly before invocation match, and tests 8-10 verify it.", + "file": "scripts/check_guards_are_wired.sh", + "grounding": "cited" + }, + { + "claim": "The check_pathonly_devdeps_unused_in_src.sh now properly handles python 3.10 environments missing tomllib by falling back to tomli, and exits with ENV rc=2 if neither is available. It also enforces SCAN-DONE trailer.", + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "grounding": "cited" + }, + { + "claim": "unwired_guards_baseline.txt includes the 3 dark guards added as set-aperture reveals, correctly explaining why check_pathonly_devdeps_unused_in_src.sh is omitted.", + "file": "scripts/unwired_guards_baseline.txt", + "grounding": "cited" + }, + { + "claim": "The check_baseline_ratchets.sh has exactly 6 case rows for NAME entries, matching the acceptance criteria perfectly.", + "file": "scripts/check_baseline_ratchets.sh", + "grounding": "cited" + } + ], + "raw_bytes": 4383, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Diff reviewed and verified against PMAT-3644. All requirements have been met without refutation:\n1. `check_guards_are_wired.sh` drops `name:` lines before matching dispatches, preventing step names from miscounting as guard invocations. Self-tests 8-10 verify this, and mutating out `_not_a_name_line` turns rows 8 and 10 RED.\n2. `check_pathonly_devdeps_unused_in_src.sh` supports `tomllib` -> `tomli` -> ENV `rc=2`, requires `SCAN-DONE` trailer, and passes 5 death rows. Reworded comments remove `cargo ` token matches so `guard_tree.sh --no-cargo` now runs it.\n3. `unwired_guards_baseline.txt` records the 3 dark guards revealed as `set-aperture` admissions owned by `check_guards_are_wired.sh`, with `lib_baseline_ratchet.sh` supporting NAME-entry admissions and 6 test case rows in `check_baseline_ratchets.sh`. No workflow files were edited.", + "findings": [ + { + "claim": "check_guards_are_wired.sh filters out name: lines prior to matching invocations in both dispatcher_wired() and unwired_in(), correctly preventing step names like \"guard_tree.sh's own case table\" from blinding the meta-guard. Passes self-test rows 8-10, and removing the drop turns rows 8 and 10 RED.", + "command": "bash /tmp/ws_test/scripts/check_guards_are_wired.sh --self-test", + "file": "scripts/check_guards_are_wired.sh", + "grounding": "measured", + "line": 80, + "output": "ok row 1 the unwired guard is reported, the wired one is not\nok row 2 wiring it clears the report\nok row 3 a guard dispatched by guard_tree.sh --no-cargo counts as wired\nok row 4 removing the dispatcher line brings the guard back as unwired\nok row 5 a guard wired only in guards-nightly.yml counts as wired\nok row 6 a guard wired in no workflow at all is still reported\nok row 7 deleting the nightly workflow brings its guard back as unwired\nok row 8 a step NAMED guard_tree.sh's ... is not a dispatch; the guard stays unwired\nok row 9 a QUOTED run: line still dispatches (the control for row 8)\nok row 10 a step named after a guard wires nothing\n\nSELF-TEST PASSED (10/10)" + }, + { + "claim": "check_pathonly_devdeps_unused_in_src.sh now tries tomllib then tomli and returns ENV rc=2 when neither is present; requires a SCAN-DONE trailer to ensure the scanner completed; passes all 5 death rows; and rewording comments eliminates CARGO_RE matches so it is included in --no-cargo execution.", + "command": "bash scripts/check_pathonly_devdeps_unused_in_src.sh --selftest", + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "grounding": "measured", + "line": 55, + "output": "ok death rc=2 both TOML readers absent (the intel shape) -> ENV rc=2\nok death rc=2 interpreter exits 1 with no output -> ENV rc=2, never 1\nok death rc=2 no interpreter at all (rc=127) -> ENV rc=2\nok death rc=0 the working interpreter, as the control -> rc=0\nok death rc=2 the whole guard under the intel shape exits ENV with no row verdict\nPASS: case table (4 must-match, 7 must-not-match, 5 death rows)" + }, + { + "claim": "check_baseline_ratchets.sh classifies unwired_guards_baseline.txt as set-aperture owned by scripts/check_guards_are_wired.sh, and lib_baseline_ratchet.sh implements NAME-entry set-aperture rules with 6 case rows in self-test.", + "command": "bash scripts/check_baseline_ratchets.sh --self-test", + "file": "scripts/check_baseline_ratchets.sh", + "grounding": "measured", + "line": 96, + "output": "PASS case table only: 61 rows (set, count, keyed, keyed2, comparand resolver,\n end-to-end, classification totality). NO baseline in this tree was\n compared — run with no arguments for that." + }, + { + "claim": "unwired_guards_baseline.txt records the 3 dark guards revealed by fixing check_guards_are_wired.sh (check_model_ladder.sh, check_pr_review_counts.sh, check_pr_review_receipt.sh) with detailed notes, while check_pathonly_devdeps_unused_in_src.sh is correctly omitted as it is now wired via guard_tree.sh --no-cargo.", + "file": "scripts/unwired_guards_baseline.txt", + "grounding": "cited", + "line": 5 + } + ], + "raw_bytes": 10261, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "docs/roadmaps/entries/PMAT-3644.yaml", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "No workflow file is edited" + ] + }, + { + "file": "scripts/check_baseline_ratchets.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The check_baseline_ratchets.sh has exactly 6 case rows for NAME entries, matching the acceptance criteria perfectly." + ] + }, + { + "file": "scripts/check_baseline_ratchets.sh", + "line": 96, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "check_baseline_ratchets.sh classifies unwired_guards_baseline.txt as set-aperture owned by scripts/check_guards_are_wired.sh, and lib_baseline_ratchet.sh implements NAME-entry set-aperture rules with 6 case rows in self-test." + ] + }, + { + "file": "scripts/check_guards_are_wired.sh", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "The name: dropping logic is wired correctly before invocation match, and tests 8-10 verify it.", + "check_guards_are_wired.sh drops name: lines before the invocation match, in dispatcher_wired() and the per-guard scan", + "rows 8-10 check the exact ci.yml step-name fixture, a quoted run: line, and a step named after a guard wires nothing", + "the mutation (remove the drop) turns rows 8 and 10 RED" + ] + }, + { + "file": "scripts/check_guards_are_wired.sh", + "line": 80, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "check_guards_are_wired.sh filters out name: lines prior to matching invocations in both dispatcher_wired() and unwired_in(), correctly preventing step names like \"guard_tree.sh's own case table\" from blinding the meta-guard. Passes self-test rows 8-10, and removing the drop turns rows 8 and 10 RED." + ] + }, + { + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "The check_pathonly_devdeps_unused_in_src.sh now properly handles python 3.10 environments missing tomllib by falling back to tomli, and exits with ENV rc=2 if neither is available. It also enforces SCAN-DONE trailer.", + "check_pathonly_devdeps_unused_in_src.sh falls back to tomli, exits with ENV rc=2, and requires a SCAN-DONE trailer", + "comments reworded so CARGO_RE no longer classifies it out of --no-cargo", + "five death rows incl. the whole guard under the intel shape" + ] + }, + { + "file": "scripts/check_pathonly_devdeps_unused_in_src.sh", + "line": 55, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "check_pathonly_devdeps_unused_in_src.sh now tries tomllib then tomli and returns ENV rc=2 when neither is present; requires a SCAN-DONE trailer to ensure the scanner completed; passes all 5 death rows; and rewording comments eliminates CARGO_RE matches so it is included in --no-cargo execution." + ] + }, + { + "file": "scripts/lib_baseline_ratchet.sh", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "set-aperture gains a NAME-entry admission with six case rows for acceptance vs refusal" + ] + }, + { + "file": "scripts/unwired_guards_baseline.txt", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "unwired_guards_baseline.txt includes the 3 dark guards added as set-aperture reveals, correctly explaining why check_pathonly_devdeps_unused_in_src.sh is omitted.", + "unwired_guards_baseline.txt records the other three dark guards as APERTURE REVEALS" + ] + }, + { + "file": "scripts/unwired_guards_baseline.txt", + "line": 5, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "unwired_guards_baseline.txt records the 3 dark guards revealed by fixing check_guards_are_wired.sh (check_model_ladder.sh, check_pr_review_counts.sh, check_pr_review_receipt.sh) with detailed notes, while check_pathonly_devdeps_unused_in_src.sh is correctly omitted as it is now wired via guard_tree.sh --no-cargo." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3651.json b/docs/audits/quorum-PMAT-3651.json new file mode 100644 index 0000000000..0b521b7ec7 --- /dev/null +++ b/docs/audits/quorum-PMAT-3651.json @@ -0,0 +1,171 @@ +{ + "ticket": "PMAT-3651", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "09ea6b5a36f6a18a40735481f81b4d6d622e4910", + "diff_sha256": "16860b2a04b90581da01cab5f669a0edcafe29e53e8a5d86bd9416c11814e007", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 15106, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly satisfies all acceptance criteria for PMAT-3651. `guard_tree.sh` correctly surfaces `UNMEASURED` and `SUMMARY` pass lines using `awk`, limiting output to 5 lines with a count for any extras, and only matches them as the first token. `check_fleet_pv_shapes_gate.sh` successfully re-prints its measured pass row with the `SUMMARY` prefix. The tests in `guard_tree_test.sh` are robust, verifying the cap, the mid-line exclusions, silent pass behavior, and importantly, including a mutation test to guarantee the assertions properly catch failures. Nothing out of scope (like closing #3567) is touched.", + "findings": [], + "raw_bytes": 3395, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly aligns with PMAT-3651. It introduces the `surface_pass_summary` awk script in `guard_tree.sh` to extract and cap the necessary lines, modifies `check_fleet_pv_shapes_gate.sh` to correctly prefix `SUMMARY` on a `PASS` verdict so it's picked up by `guard_tree`, and includes extensive tests in `guard_tree_test.sh` that assert the surfacing, the 5-line cap, the mutant failure, and ignore midline matches. No requirements are violated.", + "findings": [ + { + "claim": "The guard_tree.sh correctly surfaces lines starting with UNMEASURED or SUMMARY, indented and capped at 5 lines + count", + "file": "scripts/guard_tree.sh", + "grounding": "cited", + "line": 253 + }, + { + "claim": "check_fleet_pv_shapes_gate.sh pass row is prefixed with SUMMARY so that it reaches the log", + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "grounding": "cited", + "line": 207 + }, + { + "claim": "guard_tree_test.sh adds tests for the surfaced rows, the cap, the midline negative case, and the mutant", + "file": "scripts/tests/guard_tree_test.sh", + "grounding": "cited", + "line": 557 + } + ], + "raw_bytes": 4222, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 of 3 Independent Review for PMAT-3651: PASS. The diff satisfies all acceptance criteria in issue #3651. 1) guard_tree.sh surfaces lines starting with UNMEASURED or SUMMARY under PASS rows (indented ' ~ ', capped at 5 lines + (+N more) count). 2) check_fleet_pv_shapes_gate.sh re-prints passing rows as SUMMARY PASS verdict=... for log surfacing. 3) guard_tree_test.sh tests 22-27 pass cleanly, including non-vacuous mutation check 27. No gates weakened, no out-of-scope changes.", + "findings": [ + { + "claim": "guard_tree.sh surfaces lines starting with UNMEASURED or SUMMARY under PASS rows, capped at 5 lines plus (+N more) count", + "command": "bash scripts/tests/guard_tree_test.sh", + "file": "scripts/guard_tree.sh", + "grounding": "measured", + "line": 233, + "output": "PASS 22: a passing guard's UNMEASURED line is surfaced under its PASS row (no opt-in)\nPASS 23: a passing guard's SUMMARY line is surfaced; its other output is not\nPASS 24: the words mid-line (or as a prefix of another word) are not markers\nPASS 25: a silent passing guard prints its PASS row and nothing under it\nPASS 26: seven SUMMARY lines are capped at 5 plus a (+2 more) count\nPASS 27: mutant with the surfacing deleted loses rows 22 and 23 (the assertion can fail)\n29 checks, 0 failed" + }, + { + "claim": "check_fleet_pv_shapes_gate.sh re-prints a measured PASS row with a SUMMARY prefix so guard_tree.sh captures and surfaces it in CI logs", + "command": "bash scripts/check_fleet_pv_shapes_gate.sh --self-test", + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "grounding": "measured", + "line": 205, + "output": "=== fleet pv shapes gate: case table (stub pvs, no network, no real fleet binary) ===\nok row 1 capable pv + Pass verdict -> PASS, row names the runner\nok row 2 a measured PASS is re-printed as SUMMARY for the guard-tree log\n...\nSELF-TEST PASSED: 11 rows" + } + ], + "raw_bytes": 5912, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "line": 205, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "check_fleet_pv_shapes_gate.sh re-prints a measured PASS row with a SUMMARY prefix so guard_tree.sh captures and surfaces it in CI logs" + ] + }, + { + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "line": 207, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "check_fleet_pv_shapes_gate.sh pass row is prefixed with SUMMARY so that it reaches the log" + ] + }, + { + "file": "scripts/guard_tree.sh", + "line": 233, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "guard_tree.sh surfaces lines starting with UNMEASURED or SUMMARY under PASS rows, capped at 5 lines plus (+N more) count" + ] + }, + { + "file": "scripts/guard_tree.sh", + "line": 253, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "The guard_tree.sh correctly surfaces lines starting with UNMEASURED or SUMMARY, indented and capped at 5 lines + count" + ] + }, + { + "file": "scripts/tests/guard_tree_test.sh", + "line": 557, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "guard_tree_test.sh adds tests for the surfaced rows, the cap, the midline negative case, and the mutant" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3658.json b/docs/audits/quorum-PMAT-3658.json new file mode 100644 index 0000000000..f4b0417e61 --- /dev/null +++ b/docs/audits/quorum-PMAT-3658.json @@ -0,0 +1,136 @@ +{ + "ticket": "PMAT-3658", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "6c43aadf57ec04712f5618805cd13952da52f94e", + "diff_sha256": "6d7475db1bb74b1ba0537f9bedd743fba826abbbbe2e0c65b9382694e863dc3d", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 28103, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff successfully implements all acceptance criteria in PMAT-3658 and its amendments without introducing out-of-scope changes or weakening any rules.\n\n- The SQUASH queue handling removes the `HEAD^2` constraint, elegantly using `HEAD^1..HEAD` to re-derive the PR diff, resolving the ~1 hour cost for text-only PRs.\n- `ci.yml` modifications are strictly bounded to the two authorized places: adding a fallback log for `tier == none` and adding the `--no-renames` flag to `pull_request` touch-lists.\n- `diff_into` similarly incorporates `--no-renames` to prevent rename detections from concealing deleted source files as purely docs additions.\n- The `docs_only` implementation is precisely scoped to `docs/roadmaps/` and `docs/audits/`, enforcing that all diff paths must exist at HEAD to pass as docs-only (preventing un-tested file deletions).\n- Testing is thorough, featuring the requisite `mutant-head2.sh`, `mutant-renames.sh`, and `contains_all` assertions.\n\nAll assertions are correct. Verdict is PASS.", + "findings": [], + "raw_bytes": 5968, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements all acceptance criteria in the PMAT-3658 ticket and amendments without any unauthorized changes. \n\n1. `ci.yml` changes are strictly limited to the two authorized places: adding the log step for `tier == none` and adding `--no-renames` to the `pull_request` touch-list line.\n2. `ci_test_tier.sh` modifies `diff_into` to use `--no-renames` as required.\n3. The SQUASH queue cases (using `HEAD^1` and gracefully degrading) are fully handled and covered by case-table rows, including a mutant test for the `HEAD^2` reversion.\n4. Docs-only diffs and empty diffs correctly emit `tier=none`, well-verified by corresponding test coverage. \n5. Pushes to SQUASH landing gracefully utilize `HEAD^1`. \n\nNo rules were weakened, and all receipt claims are backed exactly by the diff logic and its tests.", + "findings": [], + "raw_bytes": 3773, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 Review: PASS. The diff correctly implements all requirements of ticket PMAT-3658 without weakening any gates or adding unrequested changes. Squash merge queue entries with one parent now correctly re-derive their test tier using HEAD^1..HEAD, docs-only diffs correctly skip workspace-test with tier=none, and --no-renames ensures both sides of renames are tracked. Self-tests pass.", + "findings": [ + { + "claim": "ci_test_tier.sh --self-test executes squash merge queue re-derivation, docs-only tier=none filtering, and --no-renames rename detection self-tests without failures.", + "command": "bash scripts/ci_test_tier.sh --self-test", + "file": "scripts/ci_test_tier.sh", + "grounding": "measured", + "line": 458, + "output": "ok row 1 rc=0 schedule -> full (FULL still lives on the nightlies — decision D-1)\nok row 2 rc=0 workflow_dispatch -> full\nok row 3 rc=2 unknown event -> ENV (exit 2), never a guess\nok row 4 rc=0 pull_request, scripts-only diff -> quick with NO crates\nok row 5 rc=0 ...and the tree-reader targets are in it (readme_contract, the reader that bit #3039)\nok row 6 rc=0 ...including the lib module whose unit test reads a baseline (aprender-contracts)\nok row 7 rc=0 pull_request, root Cargo.toml touched -> full (fail closed)\nok row 8 rc=0 pull_request over the cap (aprender-core) -> quick, NOT an hour of full workspace tests\nok row 9 rc=0 ...with check_workspace=1: ONE cargo check --workspace covers every reverse dependent (rule (i))\nok row 10 rc=0 ...and the touched crate itself still runs its tests\nok row 11 rc=0 pull_request WITHIN the cap emits NO check_workspace line (the workspace check is not free)" + }, + { + "claim": ".github/workflows/ci.yml incorporates --no-renames on git diff for pull_request touched.txt and adds an auditable step logging tier=none execution as specified in PMAT-3658.", + "file": ".github/workflows/ci.yml", + "grounding": "cited", + "line": 448 + }, + { + "claim": "ci_test_tier.sh removes the mandatory HEAD^2 check for merge_group events, allowing squash merge queues (one parent) to re-derive the PR tier from HEAD^1..HEAD instead of falling closed to tier=full.", + "file": "scripts/ci_test_tier.sh", + "grounding": "cited", + "line": 292 + } + ], + "raw_bytes": 6804, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": ".github/workflows/ci.yml", + "line": 448, + "lanes_agreeing": [ + 3 + ], + "claims": [ + ".github/workflows/ci.yml incorporates --no-renames on git diff for pull_request touched.txt and adds an auditable step logging tier=none execution as specified in PMAT-3658." + ] + }, + { + "file": "scripts/ci_test_tier.sh", + "line": 292, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "ci_test_tier.sh removes the mandatory HEAD^2 check for merge_group events, allowing squash merge queues (one parent) to re-derive the PR tier from HEAD^1..HEAD instead of falling closed to tier=full." + ] + }, + { + "file": "scripts/ci_test_tier.sh", + "line": 458, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "ci_test_tier.sh --self-test executes squash merge queue re-derivation, docs-only tier=none filtering, and --no-renames rename detection self-tests without failures." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3662.json b/docs/audits/quorum-PMAT-3662.json new file mode 100644 index 0000000000..091ec92905 --- /dev/null +++ b/docs/audits/quorum-PMAT-3662.json @@ -0,0 +1,205 @@ +{ + "ticket": "PMAT-3662", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "6b04f66a82ef40f9349d88d12faf8fe289b77866", + "diff_sha256": "4f3cb2d4adf71ae1a404f78110835ee97073997800c89a181b0e98ec139a5160", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 19429, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly addresses all four acceptance criteria of PMAT-3662.\n\n1. It replaces the hand-typed `match` arms in all three sites (`sliding_window_entropy.rs`, `validate_manifest.rs`, and `model_source.rs`) with `trueno_quant::GgmlType::from_id(...).map_or(...)` or `.map(...)`, as requested.\n2. It adds a new test per site that feeds ids 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS), and 30 (BF16), resolving correctly to the upstream names.\n3. Unknown ids (e.g. 4, 5, 999) are explicitly tested and continue to print each site's existing unknown token (`\"Unknown\"`, `\"UNKNOWN\"`, or `None`), maintaining stability.\n4. `apr-cli`'s `Cargo.toml` properly adds the workspace dependency `trueno-quant = { workspace = true }`. As noted in the roadmap facts, `aprender-serve` already compiles correctly and thus its existing dependencies suffice. No claims from the ticket are weakened or missing.", + "findings": [], + "raw_bytes": 3954, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements all four acceptance criteria of PMAT-3662 exactly as specified.\n\n1. It replaces the hand-typed match arms in `sliding_window_entropy.rs`, `validate_manifest.rs`, and `model_source.rs` with `trueno_quant::GgmlType::from_id(id).map_or(..., GgmlType::as_str)` or equivalent.\n2. It adds a test to each site that feeds the required ids `10`, `16`, `23`, and `30` and checks they map to their correct upstream names (`Q2_K`, `IQ2_XXS`, `IQ4_XS`, `BF16`).\n3. Unknown ids `4` and `5` (and `999`/`9999` either newly added or via pre-existing tests) return the site's specific unknown tokens (`\"Unknown\"`, `\"UNKNOWN\"`, and `None` respectively).\n4. `apr-cli/Cargo.toml` correctly adds `trueno-quant = { workspace = true }` as a dependency.", + "findings": [], + "raw_bytes": 3670, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "As Lane 3 of 3 independent reviewers, I have reviewed the diff against Ticket PMAT-3662. All four acceptance criteria are fully met:\n\n1. Every non-example id->name mapping in `apr-cli` (`sliding_window_entropy.rs:297`, `validate_manifest.rs:795`) and `aprender-serve` (`model_source.rs:209`) routes through `trueno_quant::GgmlType::from_id(id)`.\n2. Tests covering IDs 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS), 30 (BF16), and unknown IDs have been added for all three sites (`hex_options_detect.rs:145`, `validate_manifest.rs:1676`, `model_source.rs:304`).\n3. Unknown IDs maintain site-specific unknown tokens (\"Unknown\", \"UNKNOWN\", None).\n4. `apr-cli` workspace dependency on `trueno-quant` is added to `Cargo.toml:158` and reflected in `Cargo.lock:268`.\n\nNo refutations, weakened gates, or out-of-scope changes were found. Verdict: PASS.", + "findings": [ + { + "claim": "apr-cli Cargo.toml adds dependency trueno-quant = { workspace = true }, satisfying acceptance criterion 4.", + "file": "crates/apr-cli/Cargo.toml", + "fix": "N/A", + "grounding": "cited", + "line": 158 + }, + { + "claim": "ggml_dtype_name in apr-cli/src/commands/sliding_window_entropy.rs is routed through trueno_quant::GgmlType::from_id(dtype).map_or(\"Unknown\", trueno_quant::GgmlType::as_str), preserving the \"Unknown\" fallback token.", + "file": "crates/apr-cli/src/commands/sliding_window_entropy.rs", + "fix": "N/A", + "grounding": "cited", + "line": 297 + }, + { + "claim": "test_ggml_dtype_name_names_every_family in apr-cli/src/commands/hex_options_detect.rs verifies IDs 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS), 30 (BF16) and unknown IDs 4, 5, 999.", + "file": "crates/apr-cli/src/commands/hex_options_detect.rs", + "fix": "N/A", + "grounding": "cited", + "line": 145 + }, + { + "claim": "ggml_type_name in apr-cli/src/commands/validate_manifest.rs is routed through trueno_quant::GgmlType::from_id(t).map_or(\"UNKNOWN\", trueno_quant::GgmlType::as_str), preserving the \"UNKNOWN\" fallback token.", + "file": "crates/apr-cli/src/commands/validate_manifest.rs", + "fix": "N/A", + "grounding": "cited", + "line": 795 + }, + { + "claim": "pm008_ggml_type_name_is_the_upstream_table in apr-cli/src/commands/validate_manifest.rs verifies IDs 10, 16, 23, 30 and unknown IDs 4, 5, 999.", + "file": "crates/apr-cli/src/commands/validate_manifest.rs", + "fix": "N/A", + "grounding": "cited", + "line": 1676 + }, + { + "claim": "gguf_qtype_name in aprender-serve/src/api/model_source.rs is routed through trueno_quant::GgmlType::from_id(qtype).map(trueno_quant::GgmlType::as_str), preserving the Option return type.", + "file": "crates/aprender-serve/src/api/model_source.rs", + "fix": "N/A", + "grounding": "cited", + "line": 209 + }, + { + "claim": "qtype_names_cover_every_live_family in aprender-serve/src/api/model_source.rs verifies IDs 10, 16, 23, 30 and unknown IDs 4, 5.", + "file": "crates/aprender-serve/src/api/model_source.rs", + "fix": "N/A", + "grounding": "cited", + "line": 304 + } + ], + "raw_bytes": 7916, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/Cargo.toml", + "line": 158, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "apr-cli Cargo.toml adds dependency trueno-quant = { workspace = true }, satisfying acceptance criterion 4." + ] + }, + { + "file": "crates/apr-cli/src/commands/hex_options_detect.rs", + "line": 145, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "test_ggml_dtype_name_names_every_family in apr-cli/src/commands/hex_options_detect.rs verifies IDs 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS), 30 (BF16) and unknown IDs 4, 5, 999." + ] + }, + { + "file": "crates/apr-cli/src/commands/sliding_window_entropy.rs", + "line": 297, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "ggml_dtype_name in apr-cli/src/commands/sliding_window_entropy.rs is routed through trueno_quant::GgmlType::from_id(dtype).map_or(\"Unknown\", trueno_quant::GgmlType::as_str), preserving the \"Unknown\" fallback token." + ] + }, + { + "file": "crates/apr-cli/src/commands/validate_manifest.rs", + "line": 795, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "ggml_type_name in apr-cli/src/commands/validate_manifest.rs is routed through trueno_quant::GgmlType::from_id(t).map_or(\"UNKNOWN\", trueno_quant::GgmlType::as_str), preserving the \"UNKNOWN\" fallback token." + ] + }, + { + "file": "crates/apr-cli/src/commands/validate_manifest.rs", + "line": 1676, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "pm008_ggml_type_name_is_the_upstream_table in apr-cli/src/commands/validate_manifest.rs verifies IDs 10, 16, 23, 30 and unknown IDs 4, 5, 999." + ] + }, + { + "file": "crates/aprender-serve/src/api/model_source.rs", + "line": 209, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "gguf_qtype_name in aprender-serve/src/api/model_source.rs is routed through trueno_quant::GgmlType::from_id(qtype).map(trueno_quant::GgmlType::as_str), preserving the Option return type." + ] + }, + { + "file": "crates/aprender-serve/src/api/model_source.rs", + "line": 304, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "qtype_names_cover_every_live_family in aprender-serve/src/api/model_source.rs verifies IDs 10, 16, 23, 30 and unknown IDs 4, 5." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3670.json b/docs/audits/quorum-PMAT-3670.json new file mode 100644 index 0000000000..2eaca17e5e --- /dev/null +++ b/docs/audits/quorum-PMAT-3670.json @@ -0,0 +1,137 @@ +{ + "ticket": "PMAT-3670", + "base": "053992448", + "base_resolved": "053992448", + "base_note": "no origin/053992448 exists; judged against the local ref", + "head": "984fc76a9cff6f0dc41808e7d40ee4608baea9f1", + "diff_sha256": "ce7046a5caea951e585a96e515490e92af76e29b2d0894ceb02e0480d19e2558", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 15152, + "author": { + "model": "Opus 5 (1M context)", + "family": "claude", + "source": "measured" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly identifies and handles all fleet state errors (forjar drift, pin-mismatch, incapable tool, no JSON output, and missing Python/parser errors), converting them from `FAIL` (exit code 1) to `UNMEASURED` (exit code 0). It maintains the correct failure behavior (exit code 1) for actual gate/lint failures, ensuring the gate is not weakened while fixing the issue described in PMAT-3670. All tests match the intended behavior.", + "findings": [], + "raw_bytes": 3022, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff implements exactly what PMAT-3670 requests. It successfully replaces all occurrences of fleet-state failures (e.g. forjar drift, pin-mismatch, missing binary capabilities, and parsing environment failures) with a proper UNMEASURED output line and an exit code of 0 instead of `FAIL` with a non-zero exit code. It updates the self-test table to verify the new behaviors. I found no evidence that this weakens any gates for actual shape violations or contradicts the ticket.", + "findings": [], + "raw_bytes": 3115, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 Reviewer Verdict: PASS. The diff satisfies PMAT-3670 by correctly mapping fleet state errors (no-binary, pin-mismatch, incapable, pv-env, judge-env) to UNMEASURED (exit 0) with detailed reason tags, preventing fleet state drift from failing PR gates while preserving FAIL (exit 1) for actual shape violations. Self-test suite verified with 14 passing rows including mutation tests.", + "findings": [ + { + "claim": "The updated check_fleet_pv_shapes_gate.sh passes all 14 self-test cases, verifying that fleet state errors (pin-mismatch, no-binary, incapable, pv-env, judge-env) are UNMEASURED (exit 0) and real violations remain FAIL (exit 1).", + "command": "bash /home/noah/.gemini/antigravity-cli/scratch/lane-3-check/check_fleet_pv_shapes_gate.sh --self-test", + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "grounding": "measured", + "line": 175, + "output": "=== fleet pv shapes gate: case table (stub pvs, no network, no real fleet binary) ===\nok row 1 capable pv + Pass verdict -> PASS, row names the runner\nok row 2 planted violation (verdict=Fail) -> RED\nok row 3 0.65.2 condition (no --gate) -> UNMEASURED reason=incapable, by capability not version, never PASS\nok row 4 pin present, no fleet path has the binary -> UNMEASURED reason=no-binary, never PASS\nok row 5 no pin -> UNMEASURED row (exit 0, never a PASS line)\nok row 6 verdict=Pass but pc_shape not fired -> RED (a checker firing on nothing cannot pass)\nok row 7 first fleet path absent, second present -> resolved to the second, row names it\nok row 8 binary 0.65.2 under pin 0.68.2 -> UNMEASURED reason=pin-mismatch, names both versions and the runner, never PASS (before any lint)\nok row 9 mutant restoring FAIL on pin-mismatch exits 1 -- the pin-mismatch row would go RED under it\nok row 10 pinned pv exits 2 with no JSON -> UNMEASURED reason=pv-env with pv's stderr, never PASS\nok row 11 mutant restoring the ENV exit on pv-env exits 2 -- the pv-env row would go RED under it\nok row 12 no python3 for the verdict parser -> UNMEASURED reason=judge-env (rc=127), never PASS\nok row 13 pv without a pc_shape field -> PASS, row says control=unreported\nok row 14 pin ' 0.68.2-rc1 \\n' strips to 0.68.2-rc1 -- no tr, no busybox class bug\nSELF-TEST PASSED: 14 rows" + }, + { + "claim": "Fleet state conditions (no-binary, pin-mismatch, incapable, pv-env, judge-env) return UNMEASURED with exit code 0 rather than FAIL with exit code 1.", + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "grounding": "cited", + "line": 95 + }, + { + "claim": "Real shape violations (verdict != Pass or pc_shape control failure) still return FAIL with exit code 1.", + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "grounding": "cited", + "line": 119 + } + ], + "raw_bytes": 10682, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "line": 95, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Fleet state conditions (no-binary, pin-mismatch, incapable, pv-env, judge-env) return UNMEASURED with exit code 0 rather than FAIL with exit code 1." + ] + }, + { + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "line": 119, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Real shape violations (verdict != Pass or pc_shape control failure) still return FAIL with exit code 1." + ] + }, + { + "file": "scripts/check_fleet_pv_shapes_gate.sh", + "line": 175, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "The updated check_fleet_pv_shapes_gate.sh passes all 14 self-test cases, verifying that fleet state errors (pin-mismatch, no-binary, incapable, pv-env, judge-env) are UNMEASURED (exit 0) and real violations remain FAIL (exit 1)." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: grounding NOT COMPARED — structured_output holds a verdict but no single verdict object could be read out of .response (10682 raw bytes), so agy's labels stand unchecked for this lane (PMAT-082)" + ], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=Opus 5 (1M context)/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3682.json b/docs/audits/quorum-PMAT-3682.json new file mode 100644 index 0000000000..1c3b85952e --- /dev/null +++ b/docs/audits/quorum-PMAT-3682.json @@ -0,0 +1,235 @@ +{ + "ticket": "PMAT-3682", + "base": "c0dcf208a", + "base_resolved": "c0dcf208a", + "base_note": "no origin/c0dcf208a exists; judged against the local ref", + "head": "2237f4b95da0d300d7d13247891f7b83c3bafa8f", + "diff_sha256": "0b768cc0a1fc620dd52d4f5f99af238d315faee7e2a4db4efbe443f7aa13566e", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 11411, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly fulfills all requirements of PMAT-3682. The `git ls-files` stderr is no longer suppressed, preventing dubious ownership failures from being silently swallowed. When git fails, the script now cleanly propagates the error code 2 along with an appropriate error message indicating that git failed, ensuring no false 'holds 0' count is generated. The self-tests properly replicate this scenario and the mutation shows the intended behavior when the fix is rolled back. No ticket constraints are violated.", + "findings": [ + { + "claim": "git stderr is no longer discarded (no 2>/dev/null) and ls-files uses the exact arguments specified in the ticket.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 35 + }, + { + "claim": "count_and_check returns 2 with a message naming git upon git failure, and verify propagates this with `return $?`.", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited", + "line": 38 + }, + { + "claim": "A self-test row verifies that git dubious ownership errors yield exit 2 without saying 'holds 0'. Mutating it to the pre-fix behavior correctly turns it RED.", + "command": "cat << 'EOF' > test_script.sh\n#!/bin/bash\nset -euo pipefail\n\ntd=$(mktemp -d)\nmkdir -p \"$td/evidence/parity\"\necho 2 > \"$td/evidence/parity/EXPECTED_RECEIPTS\"\nEXPECTED_FILE=\"evidence/parity/EXPECTED_RECEIPTS\"\n\ncount_and_check_old() {\n local root=$1 records=0 unmigrated=()\n local f\n while IFS= read -r f; do\n [ -n \"$f\" ] || continue\n echo \"$f\"\n done < <(cd \"$root\" && git ls-files 'evidence/parity/*.json' 'evidence/parity/**/*.json' 2>/dev/null | sort -u)\n echo $records\n}\n\nexpected_of() {\n cat \"$1/$EXPECTED_FILE\"\n}\n\nverify_old() {\n local root=$1 measured expected\n [ -f \"$root/$EXPECTED_FILE\" ] || { printf 'FAIL %s is missing\\n' \"$EXPECTED_FILE\" >&2; return 2; }\n measured=$(count_and_check_old \"$root\") || return 1\n expected=$(expected_of \"$root\")\n if [ \"$measured\" = \"$expected\" ]; then\n printf 'PASS %s receipt(s) under evidence/parity/**, and %s says %s.\\n' \\\n \"$measured\" \"$EXPECTED_FILE\" \"$expected\"\n else\n printf 'FAIL %s says %s; the tree holds %s\\n' \"$EXPECTED_FILE\" \"$expected\" \"$measured\" >&2\n return 1\n fi\n}\n\ntest_mutant() {\n mkdir -p \"$td/nogit\"\n printf '#!/bin/sh\\necho \"fatal: detected dubious ownership in repository\" >&2\\nexit 128\\n' > \"$td/nogit/git\"\n chmod 755 \"$td/nogit/git\"\n local got=0 gout\n gout=$(PATH=\"$td/nogit:$PATH\" verify_old \"$td\" 2>&1) || got=$?\n if [ \"$got\" -eq 2 ] && grep -q 'git' <<<\"$gout\" && ! grep -q 'holds 0' <<<\"$gout\"; then\n printf 'ok git refusing the tree is ENV exit 2 naming git, never a count of 0\\n'\n else\n printf 'FAIL git refused and the result was exit %s: %s\\n' \"$got\" \"$gout\"\n fi\n}\n\necho \"Mutant:\"\ntest_mutant\nEOF\nbash test_script.sh", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 143, + "output": "Mutant:\nFAIL git refused and the result was exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0" + } + ], + "raw_bytes": 9039, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements all acceptance criteria of PMAT-3682. Git failures now produce an ENV exit 2 with a message naming git, rather than failing silently and returning 0 receipts. The verify function propagates the exit code, and a self-test verifies this behavior using a mocked git command that simulates the dubious ownership error.", + "findings": [ + { + "claim": "git stderr is no longer discarded and is executed with -c safe.directory=\"$root\" -C \"$root\"", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited" + }, + { + "claim": "A git failure returns 2 and prints a message naming git, and verify propagates this status", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited" + }, + { + "claim": "A self-test row is added that stubs git to exit 128, expects exit 2, and requires output to not contain 'holds 0'", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "cited" + } + ], + "raw_bytes": 3942, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Lane 3 of 3 review complete. The diff for PMAT-3682 fully satisfies all 4 acceptance criteria and does not weaken any gates or violate any requirements.\n1. `count_and_check` uses `git -c safe.directory=\"$root\" -C \"$root\" ls-files ...` and git stderr is no longer redirected to `/dev/null`.\n2. A git failure causes `count_and_check` to return exit status 2 (ENV) with an explicit stderr message naming `git ls-files`. `verify` propagates exit status 2 via `return $?`.\n3. The self-test includes a stub for `git` that exits 128 with a dubious ownership message, asserting exit status 2, message naming `git`, and absence of \"holds 0\".\n4. Restoring the pre-fix `count_and_check` causes the self-test row to fail (goes RED) with exit code 1.\nVerdict: PASS.", + "findings": [ + { + "claim": "count_and_check uses git -c safe.directory=\"$root\" -C \"$root\" ls-files without discarding stderr and returns exit status 2 with a message naming git when git fails.", + "command": "bash -c 'bash /home/noah/.gemini/antigravity-cli/scratch/parity_receipt_denominator.sh --print /nonexistent 2>&1 || echo \"exit: $?\"'", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 34, + "output": "ENV git ls-files failed in /home/noah/.gemini/antigravity-cli: the receipt universe cannot be read, so no count is reported\nexit: 2" + }, + { + "claim": "Self-test stub for git exiting 128 with dubious ownership message verifies exit status 2, requires git in error message, and asserts output does NOT contain \"holds 0\".", + "command": "bash /home/noah/.gemini/antigravity-cli/scratch/parity_receipt_denominator.sh --self-test", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 151, + "output": "ok a record is counted and an unrelated document is not (measured 1)\nok an unmigrated legacy record is refused by name\nok a receipt added without bumping the denominator disagrees\nok bumping the denominator makes them agree\nok git refusing the tree is ENV exit 2 naming git, never a count of 0" + }, + { + "claim": "Restoring pre-fix count_and_check causes the new self-test row to go RED with exit code 1.", + "command": "bash /home/noah/.gemini/antigravity-cli/scratch/parity_receipt_denominator_mutant.sh --self-test || echo \"exit: $?\"", + "file": "scripts/parity_receipt_denominator.sh", + "grounding": "measured", + "line": 28, + "output": "FAIL git refused and the result was exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0.\n Unknown{WrongCorpus}: update the denominator in the SAME commit as the receipt.\nexit: 1" + }, + { + "claim": "Roadmap entry docs/roadmaps/entries/PMAT-3682.yaml and docs/roadmaps/roadmap.yaml accurately define all 4 acceptance criteria for issue #3682.", + "file": "docs/roadmaps/entries/PMAT-3682.yaml", + "grounding": "cited", + "line": 1 + } + ], + "raw_bytes": 7853, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "docs/roadmaps/entries/PMAT-3682.yaml", + "line": 1, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Roadmap entry docs/roadmaps/entries/PMAT-3682.yaml and docs/roadmaps/roadmap.yaml accurately define all 4 acceptance criteria for issue #3682." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": null, + "lanes_agreeing": [ + 2 + ], + "claims": [ + "A git failure returns 2 and prints a message naming git, and verify propagates this status", + "A self-test row is added that stubs git to exit 128, expects exit 2, and requires output to not contain 'holds 0'", + "git stderr is no longer discarded and is executed with -c safe.directory=\"$root\" -C \"$root\"" + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 28, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Restoring pre-fix count_and_check causes the new self-test row to go RED with exit code 1." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 34, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "count_and_check uses git -c safe.directory=\"$root\" -C \"$root\" ls-files without discarding stderr and returns exit status 2 with a message naming git when git fails." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 35, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "git stderr is no longer discarded (no 2>/dev/null) and ls-files uses the exact arguments specified in the ticket." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 38, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "count_and_check returns 2 with a message naming git upon git failure, and verify propagates this with `return $?`." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 143, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "A self-test row verifies that git dubious ownership errors yield exit 2 without saying 'holds 0'. Mutating it to the pre-fix behavior correctly turns it RED." + ] + }, + { + "file": "scripts/parity_receipt_denominator.sh", + "line": 151, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Self-test stub for git exiting 128 with dubious ownership message verifies exit status 2, requires git in error message, and asserts output does NOT contain \"holds 0\"." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index 9e242c1221..f87c0b7e58 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -86,7 +86,7 @@ apr,apr attn-parity-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/ apr,apr attn-viz-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1020,high apr,apr audio-inspect-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:983,high apr,apr awq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:877,high -apr,apr beat-run,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:424,high +apr,apr beat-run,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:428,high apr,apr bench,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:88,high apr,apr canary check,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:44,medium apr,apr canary create,4,x86_64-linux,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/canary.rs:30,medium @@ -120,14 +120,14 @@ apr,apr chat,6,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/ext apr,apr chat --backend cpu,6,x86_64-linux,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr chat --backend cuda,6,x86_64-linux;nvidia-cuda,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium apr,apr chat --backend wgpu,6,UNKNOWN,ollama,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:85 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high -apr,apr check,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:776,high +apr,apr check,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:780,high apr,apr check-finite-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1040,low -apr,apr code,2,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:796,high -apr,apr code --input-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:799 (CodeInputFormat crates/apr-cli/src/commands_enum.rs:43),high -apr,apr code --output-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:793 (CodeOutputFormat crates/apr-cli/src/commands_enum.rs:31),high +apr,apr code,2,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:800,high +apr,apr code --input-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:803 (CodeInputFormat crates/apr-cli/src/commands_enum.rs:43),high +apr,apr code --output-format json,2,x86_64-linux,claude-code,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:797 (CodeOutputFormat crates/apr-cli/src/commands_enum.rs:31),high apr,apr compare-hf,9,x86_64-linux,safetensors,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:563,medium -apr,apr compile,9,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:678,medium -apr,apr convert,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:600,high +apr,apr compile,9,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:682,medium +apr,apr convert,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:604,high apr,apr data audit,3,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:54,high apr,apr data balance,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:126,high apr,apr data decontaminate,2,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/apr-cli/src/data_commands.rs:96,high @@ -170,12 +170,12 @@ apr,apr data x schema,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline apr,apr data x view,6,x86_64-linux,huggingface-transformers,no,4,data-pipeline,crates/aprender-data/src/cli/mod.rs:140,low apr,apr dataset audio-inspect,6,x86_64-linux,huggingface-transformers,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1440,low apr,apr ddp-metrics-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:952,high -apr,apr debug,3,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:255,high -apr,apr debug embed-viz,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:908,low +apr,apr debug,3,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:259,high +apr,apr debug embed-viz,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:912,low apr,apr decrypt,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:140,medium apr,apr devices,8,x86_64-linux;nvidia-cuda,ollama,yes,10,apr-core-commands,crates/apr-cli/src/extended_commands.rs:636,high apr,apr diagnose,9,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:833,medium -apr,apr diff,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:308,high +apr,apr diff,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:312,high apr,apr distill,9,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:143,medium apr,apr dry-sampling-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:871,high apr,apr embed,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1357,medium @@ -194,36 +194,36 @@ apr,apr eval --task mbpp,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/di apr,apr eval --task plan,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1430 (flag crates/apr-cli/src/extended_commands.rs:139),high apr,apr eval --task verify,6,x86_64-linux,NONE,no,8,rag-eval,crates/apr-cli/src/dispatch_analysis.rs:1427 (flag crates/apr-cli/src/extended_commands.rs:139),high apr,apr experiment view,6,x86_64-linux,pytorch,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1567,low -apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:443,high +apr,apr explain,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:447,high apr,apr explain-token-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1065,low -apr,apr export,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:472,high -apr,apr export --format coreml,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high -apr,apr export --format gguf,6,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high -apr,apr export --format mlx,6,x86_64-linux;nvidia-cuda,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high -apr,apr export --format onnx,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high -apr,apr export --format openvino,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high -apr,apr export --format safetensors,6,x86_64-linux;nvidia-cuda,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:477,high +apr,apr export,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:476,high +apr,apr export --format coreml,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export --format gguf,6,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export --format mlx,6,x86_64-linux;nvidia-cuda,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export --format onnx,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export --format openvino,6,x86_64-linux;nvidia-cuda,pytorch,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high +apr,apr export --format safetensors,6,x86_64-linux;nvidia-cuda,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:481,high apr,apr finetune,2,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:6,high apr,apr finetune --task classify,2,x86_64-linux,unsloth,no,10,apr-core-commands,crates/apr-cli/src/model_ops_commands.rs:48,high apr,apr flow,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:667,high apr,apr fp8-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:883,high apr,apr gbnf-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1130,high apr,apr gptq-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:895,high -apr,apr gpu,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:789,high +apr,apr gpu,6,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:793,high apr,apr gpu-memtrace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1078,low apr,apr grad-norm,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1143,high apr,apr hang-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:934,high apr,apr hex,3,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:587,high apr,apr imatrix-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1166,high -apr,apr import,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:502,high -apr,apr inspect,3,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:220,high +apr,apr import,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:506,high +apr,apr inspect,3,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:224,high apr,apr kernel parity,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1471,low apr,apr kv-timeline-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1084,high -apr,apr lint,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:409,high -apr,apr list,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:591,medium -apr,apr manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:434,medium -apr,apr mcp,1,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:768,high -apr,apr merge,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:705,high +apr,apr lint,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:413,high +apr,apr list,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:595,medium +apr,apr manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:438,medium +apr,apr mcp,1,x86_64-linux,claude-code,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:772,high +apr,apr merge,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:709,high apr,apr modelfile parse,6,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1589,low apr,apr monitor,9,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:448,medium apr,apr mono archive,6,UNKNOWN,NONE,no,10,apr-core-commands,crates/apr-cli/src/commands/mono.rs:38,high @@ -253,7 +253,7 @@ apr,apr prune,4,x86_64-linux,unsloth,yes,10,apr-core-commands,crates/apr-cli/src apr,apr ptx,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:362,high apr,apr ptx-map,6,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:330,high apr,apr publish,6,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/tool_commands.rs:46,high -apr,apr pull,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:537,high +apr,apr pull,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:541,high apr,apr pv audit,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:67,low apr,apr pv book,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:337,low apr,apr pv certify,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprender-contracts-cli/src/cli.rs:418,low @@ -295,11 +295,11 @@ apr,apr pv verify-structure,6,x86_64-linux,NONE,no,6,contracts-pv,crates/aprende apr,apr qa,6,x86_64-linux;nvidia-cuda,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:238,high apr,apr qualify,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:685,high apr,apr quant-preservation-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1204,high -apr,apr quantize,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:738,high -apr,apr quantize --scheme fp16,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:743,high -apr,apr quantize --scheme int4,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:743,high -apr,apr quantize --scheme int8,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:743,high -apr,apr quantize --scheme q4k,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:743,high +apr,apr quantize,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:742,high +apr,apr quantize --scheme fp16,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high +apr,apr quantize --scheme int4,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high +apr,apr quantize --scheme int8,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high +apr,apr quantize --scheme q4k,4,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:747,high apr,apr rag demo,6,x86_64-linux,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:107,low apr,apr rag eval compare,6,UNKNOWN,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:454,high apr,apr rag eval gate,6,UNKNOWN,huggingface-transformers,no,8,rag-eval,crates/aprender-rag-cli/src/lib.rs:465,high @@ -317,7 +317,7 @@ apr,apr react-trace-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/ apr,apr registry aliases,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/registry.rs:12,high apr,apr registry-quota-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1160,high apr,apr rerank,6,x86_64-linux,huggingface-transformers,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1252,high -apr,apr rm,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:594,medium +apr,apr rm,4,x86_64-linux,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:598,medium apr,apr rm-gc-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1185,high apr,apr rosetta chain,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:67,low apr,apr rosetta compare-inference,2,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:106,high @@ -328,9 +328,9 @@ apr,apr rosetta inspect,6,x86_64-linux,safetensors,no,10,apr-core-commands,crate apr,apr rosetta validate-stats,6,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:203,low apr,apr rosetta verify,2,x86_64-linux,safetensors,no,10,apr-core-commands,crates/apr-cli/src/commands/rosetta.rs:86,high apr,apr run,2,x86_64-linux;nvidia-cuda,llama.cpp,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:105,high -apr,apr run --backend cpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:160 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high -apr,apr run --backend cuda,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:160 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium -apr,apr run --backend wgpu,2,UNKNOWN,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:160 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high +apr,apr run --backend cpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high +apr,apr run --backend cuda,4,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),medium +apr,apr run --backend wgpu,2,UNKNOWN,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:164 (BACKEND_VALUES crates/apr-cli/src/commands_enum.rs:16),high apr,apr run --gpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:81,high apr,apr run --no-gpu,2,x86_64-linux;nvidia-cuda,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:90,high apr,apr run --stream,2,x86_64-linux;nvidia-cuda,ollama,no,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:123,high @@ -362,8 +362,8 @@ apr,apr sim run,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/sr apr,apr sim validate,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:92,low apr,apr sim verify,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:97,low apr,apr sim version,6,x86_64-linux,NONE,no,11,simulation,crates/aprender-simulate/src/cli/args.rs:119,low -apr,apr stamp,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:623,high -apr,apr tensors,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:347,high +apr,apr stamp,2,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:627,high +apr,apr tensors,4,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:351,high apr,apr test llm bench,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1670 (dispatch crates/apr-cli/src/commands/test_llm.rs:399),medium apr,apr test llm bench --band,6,x86_64-linux,vllm,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1785 (protocol crates/apr-cli/src/commands/test_llm_band.rs:348),medium apr,apr test tensor,4,x86_64-linux,NONE,no,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1622,high @@ -374,7 +374,7 @@ apr,apr tokenize plan,6,x86_64-linux,huggingface-transformers,no,10,apr-core-com apr,apr tokenize repair-manifest,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:221,low apr,apr tokenize train,6,x86_64-linux,huggingface-transformers,no,10,apr-core-commands,crates/apr-cli/src/tokenize_commands.rs:63,low apr,apr tool-use-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1124,high -apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:365,high +apr,apr trace,6,x86_64-linux,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:369,high apr,apr train apply,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:76,high apr,apr train apply --task classify,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:88,high apr,apr train apply --task pretrain,1,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:88,high @@ -388,13 +388,13 @@ apr,apr train submit,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-c apr,apr train sweep,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:201,high apr,apr train watch,6,x86_64-linux,pytorch,no,10,apr-core-commands,crates/apr-cli/src/train_commands.rs:172,low apr,apr tree,4,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:642,high -apr,apr tui,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:770,medium +apr,apr tui,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:774,medium apr,apr tune,4,x86_64-linux,pytorch,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:384,high apr,apr typical-p-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1136,high apr,apr unified-search-lint,1,x86_64-linux,NONE,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1179,high apr,apr unshard,9,x86_64-linux,llama.cpp,yes,1,apr-lint-diag,crates/apr-cli/src/extended_commands.rs:1228,medium -apr,apr validate,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:276,high -apr,apr validate-manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:294,medium +apr,apr validate,2,x86_64-linux;nvidia-cuda,safetensors,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:280,high +apr,apr validate-manifest,9,x86_64-linux,NONE,yes,10,apr-core-commands,crates/apr-cli/src/commands_enum.rs:298,medium apr,apr zram benchmark,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:47,low apr,apr zram create,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:38,low apr,apr zram remove,6,x86_64-linux,NONE,no,8,rag-eval,crates/aprender-zram-cli/src/lib.rs:41,low diff --git a/docs/roadmaps/entries/PMAT-3338.yaml b/docs/roadmaps/entries/PMAT-3338.yaml new file mode 100644 index 0000000000..e586225f79 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3338.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3338 + github_issue: 3338 + item_type: task + title: 'pv proof-status --table panics on the real corpus: a byte slice lands inside a multi-byte character (#3338)' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-21T00:32:08Z + updated: 2026-09-21T00:32:08Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Transcribed from issue #3338. Repro: `pv proof-status contracts --table` on origin/main''s contracts, rc=101, panicked at crates/aprender-contracts/src/obligation_matrix.rs:169:11: "byte index 40 is not a char boundary; it is inside ''∈'' (bytes 39..42) of `for Q4_K_M Qwen2.5-Coder, quantization ∈ {Q4_K, Q6_K}`". Plain `pv proof-status` (no --table) does not panic; only the table renderer does. Cause as the issue states it: `truncate(s, max)` returns `&s[..max]`, a byte slice, which panics when `max` falls inside a multi-byte character. Done when the table renderer no longer panics on that input — truncation cuts on a char boundary, not a byte index — and `pv proof-status contracts --table` runs on the real corpus. This fix rides in PR #3351 because that PR''s L2-column change is exercised through `--table` on the real corpus, which panicked before it.' diff --git a/docs/roadmaps/entries/PMAT-3346.yaml b/docs/roadmaps/entries/PMAT-3346.yaml new file mode 100644 index 0000000000..71ee6e8fc0 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3346.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3346 + github_issue: 3346 + item_type: task + title: 'QE2E-INV-001: ModelConstraints dropped the gated-DeltaNet shape, so 18 of every 24 Qwen3.5 layers were counted as if their tensors did not exist (#3346)' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-20T22:54:24Z + updated: 2026-09-20T22:54:24Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE, from issue #3346 ("decide which side is wrong, with a measurement") as PR #3350 answers it — the answer is neither the range nor the descriptor but the TYPE that reads the descriptor: (1) ModelConstraints carries inner_size, state_size, conv_kernel, group_count and full_attention_interval from contracts/model-families/qwen3_5.yaml, which it previously dropped; (2) the config-derived model_parameter_count equals the measured GGUF tensor sum of a REAL file — ~/models/Qwen3.5-0.8B-Q4_K_M.gguf, 320 tensors, 752,393,024 parameters, delta 0 — with the gated-DeltaNet layer (21,555,360) and the full-attention layer (18,352,640) each matching the measured blk; (3) the two shapes that contradict dense accounting are modelled from the tensors, not assumed: attn_q is [d, 2*n_h*d_k] (the q projection emits the output gate) and the DeltaNet mixer projections/conv/state norm are counted for the 18-of-24 linear layers; (4) the 9B range instantiation is NOT asserted in this PR — the repo 9b descriptor disagrees with itself (inner_size 2048 at hidden 4096; group_count 8 fails group_count*state_size == inner_size) so P(9B) computes to 8.345B, and settling it needs a real Qwen3.5-9B GGUF, recorded in the QE2E-INV-001 binding note; (5) the note edit leaves contracts/binding.yaml valid YAML (156 entries, same as main) and the tracked contracts.nt equals a fresh extraction. Out of scope: acquiring the 9B GGUF; widening the range.' diff --git a/docs/roadmaps/entries/PMAT-3351.yaml b/docs/roadmaps/entries/PMAT-3351.yaml new file mode 100644 index 0000000000..d941018672 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3351.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3351 + github_issue: 3347 + item_type: task + title: pv's L2 column reads an obligation-to-test link, not an index + status: in_progress + priority: medium + assigned_to: null + created: 2026-09-16 07:53:27+00:00 + updated: 2026-09-16 07:53:27+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE, transcribed from issue #3347 "Ask" — this row is the second and third bullets; the first bullet (write the six implementation bindings) was #3348 / roadmap PMAT-3347 and is out of scope here. Verbatim from the issue: (2) "Make the --table L2 column read the actual obligation<->test link rather than an index comparison" — the issue states the defect as: obligation_matrix.rs computes l2_tested as idx < falsification_tests.len(), so obligations showed an L2 tick before any test existed; a column that ticks on a count, not on a link, cannot report coverage. (3) "Make single-file pv lint --strict-test-binding resolve its scan root to the repo, or refuse the single-file form instead of reporting a false negative" — the issue states the defect as: it sets the scan root to the contract''s parent, contracts/, which contains no crates/, so no ref can resolve and an already-bound control contract fails identically. Either resolution named in the issue satisfies (3). No count of obligations, tests or contracts is a criterion.' diff --git a/docs/roadmaps/entries/PMAT-3370.yaml b/docs/roadmaps/entries/PMAT-3370.yaml new file mode 100644 index 0000000000..d750e8c331 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3370.yaml @@ -0,0 +1,22 @@ +- id: PMAT-3370 + github_issue: 3370 + item_type: task + title: 'EPIC: CRUX category O — AutoML parity vs AutoGluon 1.6.3 — 9 P0, 9 P1, 6 P2 (24 stories)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: docs/specifications/crux-competitive-research-ux-workflows.md + acceptance_criteria: + - 'All 24 children (#3371-#3394) closed or explicitly re-cut; apr automl fit/predict on iris and apr forecast fit on a 5-item panel run end to end; each child lands with a contract and mutation-RED evidence.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - epic + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3371.yaml b/docs/roadmaps/entries/PMAT-3371.yaml new file mode 100644 index 0000000000..d7c032a3b9 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3371.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3371 + github_issue: 3371 + item_type: task + title: 'P0: One-call tabular AutoML: fit(label) -> predict on a CSV — CRUX-O-01' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-01, competitor autogluon, demand 5. No predictor-level AutoML entry point exists. crates/aprender-core/src/automl/ is a hyperparameter TUNER (AutoTuner, TPE, GridSearch, RandomSearch, DESearch, TimeBudget) that tunes ONE estimator the caller already chose; nothing takes a labelled table and returns a fitted model. `apr train` is causal-LM pre-training only (crates/apr-cli/src/commands/train.rs:1-5) and `apr finetune --task classify` is text classification. FALSIFIABLE (contracts/crux-O-01-v1.yaml): fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline — on the iris fixture (crates/aprender-core/src/datasets/iris.csv) accuracy >= 0.90 AND the majority-class baseline is asserted at 0.333 in the same test; the same call works for a regression label without a problem_type argument — a numeric label column yields a regressor whose R^2 on a held-out split exceeds 0.5 AND the mean-predictor baseline is asserted at ~0.0; apr automl fit is reachable from the CLI — `apr automl fit --label class train.csv --out model.apr` exits 0 and writes a loadable artifact; a missing --label exits 2 with a message naming the flag. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3372.yaml b/docs/roadmaps/entries/PMAT-3372.yaml new file mode 100644 index 0000000000..b3aa11f6cf --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3372.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3372 + github_issue: 3372 + item_type: task + title: 'P0: Problem-type inference: binary / multiclass / regression / quantile from the label column — CRUX-O-02' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-02, competitor autogluon, demand 5. No function infers a task from a label column. Estimators are chosen by type name (LogisticRegression vs LinearRegression); DataFrame in crates/aprender-core/src/data/mod.rs carries ColumnStats but no label-kind classifier. FALSIFIABLE (contracts/crux-O-02-v1.yaml): two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression — a table of (label column, expected kind) fixtures including the ambiguous cases {0,1} as int, {0.0,1.0} as float, and 30 unique floats over 1000 rows all classify as documented; the override wins and a contradictory override is rejected — problem_type=regression on a string label column is an Err naming the column, not a silent cast. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3373.yaml b/docs/roadmaps/entries/PMAT-3373.yaml new file mode 100644 index 0000000000..11466513ff --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3373.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3373 + github_issue: 3373 + item_type: task + title: 'P0: Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget — CRUX-O-03' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-03, competitor autogluon, demand 5. No preset vocabulary. AutoTuner takes a SearchSpace the caller hand-builds (crates/aprender-core/src/automl/params.rs); there is no named bundle of {models, bagging, stacking, time_limit}. FALSIFIABLE (contracts/crux-O-03-v1.yaml): every preset name resolves to a portfolio and unknown names are rejected with the valid list — the five quality presets each yield a non-empty ordered model list; `presets="bestest"` is an Err whose message contains all five valid names; presets are ordered: a higher preset never fits FEWER model families than the one below it — for medium < good < high < best the family count is monotone non-decreasing, asserted pairwise. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3374.yaml b/docs/roadmaps/entries/PMAT-3374.yaml new file mode 100644 index 0000000000..2780f35fa1 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3374.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3374 + github_issue: 3374 + item_type: task + title: 'P0: Leaderboard: per-model validation/test score, fit time, predict time and stack level — CRUX-O-04' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-04, competitor autogluon, demand 5. No leaderboard type. GridSearchCVResult in model_selection/ ranks parameter settings of one estimator; TuneResult in automl/tuner.rs is a single best trial. Nothing tabulates several fitted models with timings. FALSIFIABLE (contracts/crux-O-04-v1.yaml): the leaderboard is sorted by validation score descending and its columns are fixed — columns == [model, score_val, score_test?, pred_time_val, fit_time, stack_level, fit_order] and score_val is non-increasing row to row; timings are measured, not defaulted — every fit_time and pred_time_val is > 0 after a real fit; a leaderboard built with no fit has zero rows, not zero timings; apr automl leaderboard prints the same table from a saved artifact — `apr automl leaderboard model.apr --json` emits the rows byte-equal to the in-process leaderboard. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3375.yaml b/docs/roadmaps/entries/PMAT-3375.yaml new file mode 100644 index 0000000000..b6d14f5cf0 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3375.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3375 + github_issue: 3375 + item_type: task + title: 'P0: Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) — CRUX-O-05' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-05, competitor autogluon, demand 5. Encoders exist (LabelEncoder, OneHotEncoder, OrdinalEncoder, StandardScaler, PolynomialFeatures in crates/aprender-core/src/preprocessing/) but every one is applied by hand to a column the caller already typed. There is no pass that reads a raw DataFrame, infers each column''s kind, and emits a fitted transform. Datetime expansion and text n-gram features do not exist. TfidfVectorizer exists in text/ but is not wired to a tabular pipeline. FALSIFIABLE (contracts/crux-O-05-v1.yaml): column kinds are inferred from raw values — a fixture CSV with int, float, low-cardinality string, high-cardinality string, ISO datetime and free-text columns is typed as {numeric, numeric, categorical, text, datetime, text} exactly; constant and duplicate columns are dropped and the drop is reported — a column with one unique value and an exact duplicate of another column are both absent from the transformed output AND named in the fit report; the fitted pipeline is deterministic under transform — transform(train) then transform(train) are byte-identical and transform(test) never sees a category unseen at fit as anything but the reserved unknown code. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3376.yaml b/docs/roadmaps/entries/PMAT-3376.yaml new file mode 100644 index 0000000000..948fd5b023 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3376.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3376 + github_issue: 3376 + item_type: task + title: 'P1: K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) — CRUX-O-06' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-06, competitor autogluon, demand 4. KFold and StratifiedKFold exist (model_selection/) and cross_validate scores them, but no wrapper trains one child per fold, keeps all children, averages them at predict time and exposes the out-of-fold matrix. RandomForest bags trees internally and is not reusable for other estimators. FALSIFIABLE (contracts/crux-O-06-v1.yaml): OOF predictions cover every training row exactly once — for n rows and k folds the OOF matrix has n rows, no NaN, and each row was predicted by the one child that did not see it (asserted through a fold-id trace); bagged prediction is the mean of the children — predict_proba of the bag equals the elementwise mean of the k children''s predict_proba within 1e-12. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3377.yaml b/docs/roadmaps/entries/PMAT-3377.yaml new file mode 100644 index 0000000000..0298e6a705 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3377.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3377 + github_issue: 3377 + item_type: task + title: 'P1: Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) — CRUX-O-07' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-07, competitor autogluon, demand 4. No stacking. crates/aprender-core/src/stack/ is a deployment-health module (StackHealth, InferenceConfig), not a model stacker. ensemble/ holds MixtureOfExperts gating only. FALSIFIABLE (contracts/crux-O-07-v1.yaml): level-2 features are OOF, never in-sample — a mutation that feeds in-sample level-1 predictions to level 2 is detected by the leak test: the L2 holdout score on a pure-noise label rises above chance (asserted RED) while the OOF path stays at chance; dynamic stacking falls back when stacking hurts — on a fixture where L2 holdout score < L1 holdout score, the final model is the L1 ensemble and the decision is recorded in the fit summary. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3378.yaml b/docs/roadmaps/entries/PMAT-3378.yaml new file mode 100644 index 0000000000..762d7bcec4 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3378.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3378 + github_issue: 3378 + item_type: task + title: 'P0: Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) — CRUX-O-08' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-08, competitor autogluon, demand 5. No ensemble-selection algorithm. MixtureOfExperts learns a gating network (ensemble/moe.rs); nothing performs the forward greedy selection with replacement over base-model validation predictions that yields non-negative weights summing to 1. FALSIFIABLE (contracts/crux-O-08-v1.yaml): weights are a probability vector and the ensemble never scores below its best member — sum(w)=1, all w>=0, and validation metric(ensemble) >= max over members within 1e-9, on 3 fixtures; selection is greedy with replacement and reproducible — with ensemble_size=25 the weight of a member equals its selection count / 25; two runs on the same inputs produce identical weights. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3379.yaml b/docs/roadmaps/entries/PMAT-3379.yaml new file mode 100644 index 0000000000..b2139bc2fc --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3379.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3379 + github_issue: 3379 + item_type: task + title: 'P0: Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share — CRUX-O-09' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-09, competitor autogluon, demand 5. TimeBudget and EarlyStopping exist in automl/tuner.rs but budget ONE tuner. There is no allocation of a global limit across an ordered portfolio, no per-model time share, and no ''skip the rest'' when the budget is exhausted. FALSIFIABLE (contracts/crux-O-09-v1.yaml): the wall-clock of fit never exceeds time_limit by more than the tolerance — with time_limit=5s on a portfolio that would take >60s unconstrained, elapsed <= 5s + 1s AND at least one model was skipped with reason=budget in the summary; the budget is redistributed when a model finishes early — a model that uses 10% of its share returns the remainder to the pool; the next model''s share is asserted larger than the naive equal split. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3380.yaml b/docs/roadmaps/entries/PMAT-3380.yaml new file mode 100644 index 0000000000..f141de9e6b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3380.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3380 + github_issue: 3380 + item_type: task + title: 'P1: Predictor-level permutation feature importance with p-values and confidence intervals — CRUX-O-10' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-10, competitor autogluon, demand 4. PermutationImportance exists in crates/aprender-core/src/interpret/ for a single estimator. Missing: the predictor-level call on raw (pre-pipeline) columns, num_shuffle_sets repeats, stddev / p-value / p99 columns, and subsampling. AutoGluon 1.6 also cut this call''s memory 25x (#5645). FALSIFIABLE (contracts/crux-O-10-v1.yaml): importance is reported per raw input column — on a fixture whose datetime column expands to 4 features, the importance table has one row for the datetime column, not four; a pure-noise column has importance statistically indistinguishable from zero — with num_shuffle_sets=10 the noise column''s p-value > 0.05 AND the signal column''s p-value < 0.01 on the same run. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3381.yaml b/docs/roadmaps/entries/PMAT-3381.yaml new file mode 100644 index 0000000000..ea5cded554 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3381.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3381 + github_issue: 3381 + item_type: task + title: 'P1: Decision-threshold calibration for binary metrics (calibrate_decision_threshold) — CRUX-O-11' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-11, competitor autogluon, demand 4. Probability calibration exists (PlattScaling, IsotonicRegression, TemperatureScaling in calibration.rs) but nothing searches the decision threshold that maximises f1 / balanced_accuracy / mcc on validation data and stores it on the predictor (`decision_threshold`, `set_decision_threshold`). FALSIFIABLE (contracts/crux-O-11-v1.yaml): the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search — on an imbalanced fixture (5% positives) f1 at the calibrated threshold > f1 at 0.5 by >= 0.05 absolute; the threshold is persisted with the model — save then load reproduces predict() bit-identically including the threshold; a mutation that resets the threshold to 0.5 on load turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3382.yaml b/docs/roadmaps/entries/PMAT-3382.yaml new file mode 100644 index 0000000000..9a08ae24a7 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3382.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3382 + github_issue: 3382 + item_type: task + title: 'P2: refit_full: retrain the selected models on train+validation after model selection — CRUX-O-12' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-12, competitor autogluon, demand 3. No refit-on-full-data step. cross_validate and grid_search return scores; the model returned is the one fitted on a fold, not on all rows. FALSIFIABLE (contracts/crux-O-12-v1.yaml): the refit model saw every row — the refit estimator''s training-row count equals n_train + n_val, asserted through the fitted row count it reports; refit keeps the selected hyperparameters and drops the bag children — the artifact after refit has one child per selected model and its hyperparameters are byte-equal to the pre-refit winner''s. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3383.yaml b/docs/roadmaps/entries/PMAT-3383.yaml new file mode 100644 index 0000000000..82060a6508 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3383.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3383 + github_issue: 3383 + item_type: task + title: 'P2: Model distillation: compress the ensemble into one fast student — CRUX-O-13' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-13, competitor autogluon, demand 3. Distillation exists only for LLMs in the training crate (knowledge distillation of transformers); nothing trains a single tabular student on the teacher ensemble''s soft labels with data augmentation. FALSIFIABLE (contracts/crux-O-13-v1.yaml): the student is faster and within tolerance of the teacher — student pred_time < 0.25 x teacher pred_time AND student validation score >= teacher score - 0.02 on a fixture; augmentation produces rows the training set does not contain — with augment_method=spunge the student''s training set has > n_train rows and the added rows are not row-equal to any original. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3384.yaml b/docs/roadmaps/entries/PMAT-3384.yaml new file mode 100644 index 0000000000..3de567d492 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3384.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3384 + github_issue: 3384 + item_type: task + title: 'P1: Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file — CRUX-O-14' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-14, competitor autogluon, demand 4. Single estimators serialize to .apr (bundle/, serialization/). No artifact holds a fitted feature pipeline (O-05) plus several models plus ensemble weights plus a decision threshold and loads them as one predictor. Nothing prunes non-selected models from disk. FALSIFIABLE (contracts/crux-O-14-v1.yaml): a deployment clone is smaller and predicts identically — clone_for_deployment artifact bytes < 0.5 x full artifact bytes AND predict() on the clone is bit-identical to predict() on the original for the test fixture; the artifact is self-describing and refuses a schema drift — load() on an artifact whose feature schema disagrees with the input columns is an Err naming the first mismatched column, not a wrong prediction. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3385.yaml b/docs/roadmaps/entries/PMAT-3385.yaml new file mode 100644 index 0000000000..9220aad0e4 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3385.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3385 + github_issue: 3385 + item_type: task + title: 'P2: Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size) — CRUX-O-15' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-15, competitor autogluon, demand 3. No model or ensemble is ever excluded for being slow at predict time; leaderboard timings (O-04) do not exist to compare against. FALSIFIABLE (contracts/crux-O-15-v1.yaml): the selected ensemble respects the per-row latency limit — with infer_limit=L the measured pred_time per row of the final model <= L AND at least one faster-but-worse model was preferred over a slower-but-better one (asserted on a fixture built to force the trade). Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3386.yaml b/docs/roadmaps/entries/PMAT-3386.yaml new file mode 100644 index 0000000000..6bf9ce886c --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3386.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3386 + github_issue: 3386 + item_type: task + title: 'P2: Fit diagnostics: fit_summary, model_failures and learning curves — CRUX-O-16' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-16, competitor autogluon, demand 3. ProgressCallback in automl/tuner.rs streams trial results; there is no post-fit summary object listing models trained, models failed with their error, per-model hyperparameters and per-iteration validation curves. FALSIFIABLE (contracts/crux-O-16-v1.yaml): a model that raises during fit is recorded, not swallowed and not fatal — a portfolio containing a deliberately failing model finishes; model_failures() has exactly one row naming the model and the error string; the leaderboard omits it; learning curves have one point per boosting/epoch iteration — for a GBM with n_estimators=50 and learning_curves=true the curve has 50 validation points, monotone non-increasing after early-stopping''s best iteration is asserted absent. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3387.yaml b/docs/roadmaps/entries/PMAT-3387.yaml new file mode 100644 index 0000000000..32e0d5c86f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3387.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3387 + github_issue: 3387 + item_type: task + title: 'P1: Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) — CRUX-O-17' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-17, competitor autogluon, demand 4. No tabular in-context learner. nn/ and the inference crates run causal LMs; nothing consumes (X_train, y_train, X_test) as one context and predicts without gradient steps. AutoGluon 1.4-1.6 added TabPFNv2/2.5/2.6/3, TabICL/v2, Mitra, TabDPT, Nori and made them the extreme preset. FALSIFIABLE (contracts/crux-O-17-v1.yaml): zero-gradient prediction matches a pinned reference within tolerance — loading a pinned small checkpoint via apr pull and predicting a 100-row fixture matches the committed reference probabilities within 1e-4; the context limit is enforced, not silently truncated — a context of rows > the model''s documented max is an Err naming the limit; a mutation that truncates instead turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3388.yaml b/docs/roadmaps/entries/PMAT-3388.yaml new file mode 100644 index 0000000000..6b6c698bcd --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3388.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3388 + github_issue: 3388 + item_type: task + title: 'P2: Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit — CRUX-O-18' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-18, competitor autogluon, demand 3. No estimator reports an expected peak memory before fitting and nothing checks a limit. AutoGluon 1.6 spent 4 PRs on this (#5757, #5768, #5791, GPU budgeting for parallel folds). FALSIFIABLE (contracts/crux-O-18-v1.yaml): a model whose estimate exceeds the limit is skipped with reason=memory — with memory_limit=64MiB on a fixture where the estimate for the largest model is > 64MiB, that model is absent from the leaderboard and present in the skip list with its estimate; the estimate is not a constant — the estimate for a 10x larger fixture is asserted larger than for the base fixture; a mutation returning a constant turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3389.yaml b/docs/roadmaps/entries/PMAT-3389.yaml new file mode 100644 index 0000000000..acb5bc1a8f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3389.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3389 + github_issue: 3389 + item_type: task + title: 'P0: Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq — CRUX-O-19' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-19, competitor autogluon, demand 5. crates/aprender-core/src/time_series/mod.rs is one struct, ARIMA, on one f32 series (fit/forecast/order). There is no panel container keyed by item and timestamp, no frequency, no horizon-first API and no per-item forecast. FALSIFIABLE (contracts/crux-O-19-v1.yaml): a panel of N items forecasts N x prediction_length rows — on a fixture of 5 items with 200 hourly points each and prediction_length=24 the output has exactly 120 rows, each (item_id, timestamp) unique, timestamps continuing each item''s last stamp at freq=h; irregular timestamps are rejected or regularised, never silently misaligned — an item with a missing hour is an Err naming the item and gap unless fill=forward is passed, in which case the filled row is flagged. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3390.yaml b/docs/roadmaps/entries/PMAT-3390.yaml new file mode 100644 index 0000000000..80cffcbefa --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3390.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3390 + github_issue: 3390 + item_type: task + title: 'P0: Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) — CRUX-O-20' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-20, competitor autogluon, demand 5. ARIMA.forecast returns a point path; metrics/probabilistic.rs holds classification-probability metrics, and metrics/regression.rs has no seasonal-scaled (MASE/RMSSE) or quantile (WQL/MQL) losses. FALSIFIABLE (contracts/crux-O-20-v1.yaml): quantile forecasts are monotone in the quantile level — for every (item, step) q0.1 <= q0.5 <= q0.9, asserted over the whole fixture; a mutation that shuffles the quantile columns turns this RED; each metric matches the AutoGluon reference value on a pinned fixture — WQL, MASE, SMAPE, RMSSE, WAPE and MQL computed on a committed (y_true, y_pred, quantiles) fixture equal the reference values produced by autogluon.timeseries.metrics within 1e-9. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3391.yaml b/docs/roadmaps/entries/PMAT-3391.yaml new file mode 100644 index 0000000000..c70550ddc6 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3391.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3391 + github_issue: 3391 + item_type: task + title: 'P1: Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions — CRUX-O-21' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-21, competitor autogluon, demand 4. No time-aware validation. KFold shuffles rows; nothing cuts the last k horizons of each item as expanding-window validation sets, and there is no API that returns the validation forecasts for inspection. FALSIFIABLE (contracts/crux-O-21-v1.yaml): validation windows never contain a timestamp later than the training cut for that window — for num_val_windows=3 and val_step_size=h, each window''s max training timestamp < min validation timestamp per item, asserted for every item and window; backtest_predictions rows align with backtest_targets — the two frames have identical (window, item_id, timestamp) index sets and no NaN targets. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3392.yaml b/docs/roadmaps/entries/PMAT-3392.yaml new file mode 100644 index 0000000000..8738ebd8e1 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3392.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3392 + github_issue: 3392 + item_type: task + title: 'P2: Known covariates, past covariates and static features in forecasting — CRUX-O-22' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-22, competitor autogluon, demand 3. ARIMA is univariate. No container carries per-item static features or time-varying covariates, and no model consumes them. FALSIFIABLE (contracts/crux-O-22-v1.yaml): a known covariate that fully determines the target is used — on a fixture where y = 10*holiday + noise, a model given known_covariates has MASE < 0.5 x the same model without them; future covariates missing for the horizon is an error — predict() without known_covariates for all prediction_length steps is an Err naming the first missing timestamp. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3393.yaml b/docs/roadmaps/entries/PMAT-3393.yaml new file mode 100644 index 0000000000..a8a4d54254 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3393.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3393 + github_issue: 3393 + item_type: task + title: 'P1: Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston — CRUX-O-23' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-23, competitor autogluon, demand 4. ARIMA exists (fixed order; no auto-order search). SeasonalNaive, ETS, Theta and the intermittent-demand family (Croston, ADIDA, IMAPA) are absent. AutoGluon runs these as its ''local'' tier (timeseries/models/local/) and they anchor every leaderboard. FALSIFIABLE (contracts/crux-O-23-v1.yaml): SeasonalNaive is exactly the lag-m copy — forecast[t] == y[t - m] for the whole horizon on a fixture with m=24, byte-equal; AutoARIMA selects the planted order — on a fixture generated from ARIMA(2,1,1) the selected (p,d,q) equals (2,1,1) in >= 9 of 10 seeds; ETS and Theta match the statsforecast reference — point forecasts on the AirPassengers fixture equal the statsforecast reference within 1e-6. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3394.yaml b/docs/roadmaps/entries/PMAT-3394.yaml new file mode 100644 index 0000000000..446d1c9168 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3394.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3394 + github_issue: 3394 + item_type: task + title: 'P1: Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning — CRUX-O-24' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-24, competitor autogluon, demand 4. apr pull fetches LLM checkpoints (Category A) and the inference crates run decoder transformers, but no forecasting head exists: nothing tokenises a numeric series into the model''s input, samples a horizon, and maps it back to quantiles. FALSIFIABLE (contracts/crux-O-24-v1.yaml): zero-shot output matches a pinned reference — a pinned small checkpoint pulled by `apr pull` forecasts the committed fixture within 1e-3 of the reference quantiles; fine-tuning changes the weights and improves the in-domain metric — after fine_tune on the fixture the WQL improves by >= 5% relative AND at least one weight tensor differs from the pulled checkpoint. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null diff --git a/docs/roadmaps/entries/PMAT-3401.yaml b/docs/roadmaps/entries/PMAT-3401.yaml new file mode 100644 index 0000000000..3df5483e33 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3401.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3401 + github_issue: 3401 + item_type: task + title: 'Admit AutoGluon 1.6.3 to CRUX as category O: 24 DRAFT contracts crux-O-01..24 with LIVE-PENDING gates (no implementation, no CLI, no tests), CRUX_COMPETITORS 14->15 in validator.rs + both crux_intake_tests.rs lists, CATEGORY_NAMES[O] in scripts/crux_scaffold_contracts.py, 24 master-registry rows + evidence_sources.autogluon, spec v2.3 §3/§5/§6, evidence/crux/autogluon/ (4 files), 26 roadmap fragments (PMAT-3370 epic, PMAT-3371..3394 stories, PMAT-3401 this ticket) + regenerated roadmap.yaml, and the three corpus derivatives (census, contracts.nt, README CONTRACT_COUNT) regenerated to the merged tree — child of #3370' + status: in_progress + priority: critical + assigned_to: null + created: 2026-09-16T16:30:00Z + updated: 2026-09-16T16:30:00Z + spec: docs/specifications/crux-competitive-research-ux-workflows.md + acceptance_criteria: + - 'Exactly this inventory, nothing more: contracts/crux-O-01..24-v1.yaml (24 files, registry: false, every gate LIVE-PENDING); crates/aprender-contracts/src/schema/validator.rs (CRUX_COMPETITORS 14->15, + autogluon with rationale) and crux_intake_tests.rs (autogluon in the corpus-vocabulary list and in the not-a-BEAT-pillar list); scripts/crux_scaffold_contracts.py (CATEGORY_NAMES O); contracts/crux-competitive-research-ux-v1.yaml (24 stories rows, coverage_intake 43/77/171/291, evidence_sources.autogluon, autogluon reference); docs/specifications/crux-competitive-research-ux-workflows.md (v2.3 header, §3 rows for N and O, §5 Category O table, §6 note); evidence/crux/autogluon/{README.md,api-surface.md,hello.sh,readme-verbs.txt}; docs/roadmaps/entries/PMAT-3370.yaml + PMAT-3371..3394.yaml + PMAT-3401.yaml (26 fragments) and docs/roadmaps/roadmap.yaml regenerated by the aggregator; README.md CONTRACT_COUNT 1842->1866. Absent on purpose: library code, apr CLI wiring, test bodies — those are the 24 children of #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: 'ACCEPTANCE, stated as invariants and not as counts (the first version of this row pinned "README CONTRACT_COUNT 1842->1866" and "coverage_intake 267->291" from the 2026-09-16 tree; main moved and the numbers described a tree that no longer existed, which is what the AD-04 quorum on #3395 round 0 correctly refused): (1) 24 new contract files contracts/crux-O-01-v1.yaml .. crux-O-24-v1.yaml, each metadata.category O, competitor AutoGluon, status draft, intake_status missing, registry false, and each passing pv validate; (2) CRUX_COMPETITORS grows by exactly one entry (autogluon) in crates/aprender-contracts/src/schema/validator.rs and in both crux_intake_tests.rs lists; (3) CATEGORY_NAMES gains O in scripts/crux_scaffold_contracts.py; (4) the master registry gains one row per new contract and evidence_sources.autogluon; (5) 26 roadmap fragments exist (PMAT-3370, PMAT-3371..PMAT-3394, PMAT-3401) and docs/roadmaps/roadmap.yaml equals aggregate(entries/), additive vs origin/main (deleted=0, reserialised=0); (6) contracts/census.json n_files equals origin/main''s n_files at merge time plus 24, contracts/contracts.nt equals a fresh pv extract of the merged tree, and README''s CONTRACT_COUNT blocks equal census n_files — the three derivatives ride in this PR because adding a contract stales all three (#3569). Known and NOT in scope: the census files these 24 under kind kernel because they carry no metadata.kind and registry:false (main''s crux contracts carry registry:true and file under registry); a draft/intake bucket for the census is a census question (#3640-class), not this PR''s.' diff --git a/docs/roadmaps/entries/PMAT-3495.yaml b/docs/roadmaps/entries/PMAT-3495.yaml new file mode 100644 index 0000000000..415d38c510 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3495.yaml @@ -0,0 +1,16 @@ +- id: PMAT-3495 + github_issue: 3495 + item_type: task + title: 'VERIFY-001: run the 132 Kani harnesses in CI (proof credit from runs, not declarations) and pilot Verus on one dequant/parser function' + status: planned + priority: medium + assigned_to: null + created: 2026-09-18T16:43:48Z + updated: 2026-09-18T16:43:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-3496.yaml b/docs/roadmaps/entries/PMAT-3496.yaml new file mode 100644 index 0000000000..ead822a1de --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3496.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3496 + github_issue: 3495 + item_type: task + title: 'Register PMAT-3495 (VERIFY-001 on 0.71.0) in the roadmap so the AD-04 quorum for PR #3496 can run (registration only)' + status: planned + priority: medium + assigned_to: null + created: 2026-09-21T01:39:59Z + updated: 2026-09-21T01:39:59Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:triage + notes: 'ACCEPTANCE (this row is PR #3496 itself, a roadmap-only registration): (1) docs/roadmaps/entries/PMAT-3495.yaml exists with id PMAT-3495, github_issue 3495, and a title that transcribes issue #3495''s title VERBATIM, character for character: "VERIFY-001: run the 132 Kani harnesses in CI (proof credit from runs, not declarations) and pilot Verus on one dequant/parser function" (read from `gh issue view 3495 --json title` on 2026-09-21; the issue has no rename events); (2) docs/roadmaps/roadmap.yaml equals aggregate(entries/) and the diff vs origin/main is additive only (deleted=0, reserialised=0) after the merges of origin/main; (3) the PR touches nothing outside docs/roadmaps/ and docs/audits/. NOT in scope: any Kani harness, CI wiring or proof — those are PMAT-3495''s own work and are judged when it lands. A lane that finds the entry mis-transcribed, missing or invented FAILs this row.' diff --git a/docs/roadmaps/entries/PMAT-3577.yaml b/docs/roadmaps/entries/PMAT-3577.yaml index 4ef5d6a57e..5fb0cb0987 100644 --- a/docs/roadmaps/entries/PMAT-3577.yaml +++ b/docs/roadmaps/entries/PMAT-3577.yaml @@ -14,4 +14,4 @@ estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered; pmat work add derived neither spec: nor acceptance_criteria: — defect paiml-mcp-agent-toolkit#1414). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). Done when: contracts/parity-receipt-v1.yaml with its shape on origin/main; extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; ONT-4c3 bound with the parity receipt as focus node; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml.' + notes: 'ACCEPTANCE (transcribed from gh#3577, incl. the dated AMENDMENT of 2026-09-21 02:00Z which supersedes the original item 1; §6 1e — the fragment transcribes the issue, it does not edit it). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). ITEM 1 AS AMENDED, verbatim: "contracts/parity-receipt-v2.yaml carries the shape (the constraints table below, unchanged) in its shape: block, exported to contracts/shapes.ttl. contracts/parity-receipt-v1.yaml is the SHAPELESS superseded form and says why in its own text: no v1 instance exists after the back-fill, and a shape over zero instances is a vacuous pass. A v1 record is refused BY NAME by the extractor (extract:parity-receipt) and, independently, by scripts/parity_receipt_denominator.sh — two readers, both wired (#3600 ci.yml step). A future v1 instance is therefore a refusal, not a silent pass." WHY AMENDED: the original item 1 said "parity-receipt-v1.yaml with its shape" and predates the migration — all 7 records went to v2 in the same commit, so v1 has ZERO instances, and a shape over a class nothing instantiates passes vacuously, which is the defect #3610 names and #3622 makes the shapes gate DECLINE on. Following it literally would add in one PR the thing the other PR exists to refuse. Both pro lanes read the original correctly and failed the PR on it; the ruling is on the acceptance, not the lanes. ONT-4c3 LEDGER BINDING: that ledger lives in paiml/infra (ONT-001 v4.10, infra#831) and CANNOT be done from this repository. The aprender-side deliverable is extractor + shape + denominator + receipts; the ledger row is infra s. The acceptance reads "bound in the ONT-001 ledger" as infra s row, CITED, not as a change here. REMAINING done_when, unchanged: extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml; and do NOT add a v1 shape — see the amendment.' diff --git a/docs/roadmaps/entries/PMAT-3601.yaml b/docs/roadmaps/entries/PMAT-3601.yaml new file mode 100644 index 0000000000..cb58beb6c5 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3601.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3601 + github_issue: 3601 + item_type: task + title: 'apr inspect has no byte-size entry for any IQ-family ggml type — aprender-core sizes and names ggml types from trueno_quant::TRAITS' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:47:55Z + updated: 2026-09-21T05:47:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3601 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). Done when: (1) apr inspect succeeds on a GGUF containing any of the 10 listed ids (repro: Qwen3.5-4B-UD-Q4_K_XL.gguf, dtype 23). (2) A regression test pins all 10 ids against shape.rs''s byte_size, analogous to ggml_type_table.rs''s spot_checks_against_ggml_sizes, so a future edit can''t silently drop one back out. (3) The error message, if a truly unknown id is ever hit, names the id and points at contracts/apr-cli-commands-v1.yaml''s inspect docs or similar — not a bare "Unsupported dtype N". DEVIATION FROM THE ISSUE''S FIX PROSE (not its done_when): the issue proposed porting rows from aprender-serve''s GGML_TYPES because "the unified TRAITS crate #3430 proposes doesn''t exist yet"; #3581 has since landed it (aprender-quant ggml_type.rs TRAITS[43], upstream-extracted, fixture-checked) and aprender-core already depends on it, so shape.rs reads TRAITS instead of becoming a fifth hand copy. ALSO IN THIS DIFF (same defect class, measured on apr 0.68.2 00e5d2dde): (a) apr tensors sized tensors from a second hand-typed table in aprender-core (safetensors.rs ggml_dtype_element_size SIZES) with seven IQ rows wrong and the 26-30 tail misordered (BF16 0.375 bytes/elem) — it refused the valid Qwen3.5-0.8B-UD-IQ2_XXS.gguf as "Truncated GGUF … 167936 bytes too short"; now type_size/blck_size from TRAITS. (b) ggml_dtype_name (a third copy, ids 0-30) reads TRAITS so TQ1_0/TQ2_0 are named rather than "unknown"; contracts/apr-inspect-dtype-naming-v1.yaml GGML_NAMES + KANI bound amended to match. Behaviour preservation: every id sized before keeps its exact byte count (floor arithmetic kept; characterization test over non-block-multiple counts), ids 0-30 keep their names. The IQ fabrication this sizing exposes is PMAT-3656 (#3656), fixed in the same diff.' diff --git a/docs/roadmaps/entries/PMAT-3602.yaml b/docs/roadmaps/entries/PMAT-3602.yaml new file mode 100644 index 0000000000..f5147d0c6f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3602.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3602 + github_issue: 3602 + item_type: task + title: 'apr run --gpu reported a CPU fallback as success: registry::after_generation had no production caller' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-21T00:00:00Z + updated: 2026-09-21T00:00:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3602 done_when; pmat work add derives neither spec: nor acceptance_criteria: — paiml-mcp-agent-toolkit#1414). Item 1: the GPU rejection is surfaced on a stream a user reads AND in --json. Item 2: SATISFIED BY #3606 — validate_ms instruments validate_gpu_first_token, measured 15.1s on qwen2.5-coder-0.5b; not owed again here. Item 3: root cause named for the dense path — ANSWERED, it is FP8 PREFILL, established by a 2x2 over FP8_PREFILL x FP8_DECODE with both flags named in every cell (divergence tracks FP8_PREFILL exactly, independent of FP8_DECODE; 34.3s used_gpu=false vs 16.9s used_gpu=true); a single-knob bisect could not separate them because detect_fp8_decode falls through to fp8_prefill. Item 4: the fail-vs-fall-back decision RECORDED — it already was, in registry::after_generation (R-0b, gh#3002/#3042): forced accelerator that fell to CPU = refusal exit 14, default selection = corrective line. The defect was that it had NO PRODUCTION CALLER; nor do registry::announce or registry::parity_line, which is why a real run prints zero selected: and zero parity: lines. Item 5: falsifier both ways — 6 cases in run_tests_accel_reconcile.rs; planting the pre-fix behaviour turns the two defect tests RED while the healthy-GPU control stays green. Measured on lambda RTX 4090 sm_89, apr 0.68.2 (e6f77c98c). NOT IN SCOPE, deliberately: the rejection REASON (cosine, position) is not plumbed into --json — it is produced inside realizar F2 and no channel carries it to the CLI; that is a #3606-shaped follow-up, not a value to approximate. STOP: do not widen f2_should_retry_without_fp8 here — that band belongs to gh#3483, reopened with the contradicting measurement. BLAST RADIUS: exit code changes on a shipped surface; 14 files under scripts/ and .github/workflows/ pass --gpu, incl. perf_gate.sh, parity_host_receipt.sh, model_ladder.sh — CI going red on one is the measurement, and its fix belongs on that gate.' diff --git a/docs/roadmaps/entries/PMAT-3604.yaml b/docs/roadmaps/entries/PMAT-3604.yaml new file mode 100644 index 0000000000..cd1f49fe26 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3604.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3604 + github_issue: 3604 + item_type: task + title: 'f2_validate_qwen35 runs a full CPU reference forward on EVERY call (67% of a 14s TTFT) — run it once per (model sha256, apr version, device) and receipt it' + status: planned + priority: critical + assigned_to: null + created: 2026-09-20T22:25:52Z + updated: 2026-09-20T22:25:52Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — transcribed from issue #3604 done_when (minted by the cop so the AD-04 quorum for PR #3634 can run; pmat work status refused on a missing item). (1) Validation runs once per (model sha256, apr version, device); subsequent runs of the same triple read the receipt. (2) --revalidate forces a fresh validation and rewrites the receipt. (3) Falsifier: a planted receipt carrying the wrong model sha256 must cause re-validation, not a skip. Same for a receipt written by a different apr version, and one written for a different device. Three planted-receipt cases, each asserted to re-validate. (4) A missing or unreadable receipt validates — absence is never consent. (Unknown/skip is not an option here; the fallback is to do the work.) (5) validate_ms in apr run --json (#3598) reports the cached case distinguishably from the fresh case, so the saving is readable from the receipt rather than inferred from a stopwatch. (6) Measured before/after TTFT on the 144-word row, on the same box with GPU occupancy recorded at start. ADMISSION: 0.69 if it is green by the cut, 0.70 otherwise — readiness rule, not date pressure. EXPLICITLY OUT OF SCOPE (three questions the instrument lane raised and did not answer; none block this row and none should be folded in): whether the CPU reference or the GPU probe dominates the 9.5 s (timed as one unit); whether a cheaper equivalent guard exists; the flat ~1.18 s unattributed_ms residual — recorded rather than absorbed into a neighbour, and the next stage to name. Refs #3596, #3598, #3080.' diff --git a/docs/roadmaps/entries/PMAT-3605.yaml b/docs/roadmaps/entries/PMAT-3605.yaml new file mode 100644 index 0000000000..a23ec665f6 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3605.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3605 + github_issue: 3605 + item_type: task + title: 'removed_by gets a shape: refusal-receipt-v1 with a closed escape set' + status: in_progress + priority: medium + assigned_to: null + created: 2026-09-21T00:00:00Z + updated: 2026-09-21T00:00:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3605 done_when; see paiml-mcp-agent-toolkit#1414). Done when: contracts/refusal-receipt-v1.yaml exists with a shape over removed_by, and the value space is a CLOSED SET rather than free text: pattern ^(v[0-9]+\\.[0-9]+|never|unscheduled)$. The point of the closed set is that legitimately-nothing stays SAYABLE and CHECKABLE — never and unscheduled are honest answers with a shape behind them, instead of a required field manufacturing a fake version. Fixtures cover each arm of the alternation plus a rejecting case. STOP: do not admit free text; do not make the field optional, which would make absence indistinguishable from a deliberate never.' diff --git a/docs/roadmaps/entries/PMAT-3618.yaml b/docs/roadmaps/entries/PMAT-3618.yaml new file mode 100644 index 0000000000..b53ffdb278 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3618.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3618 + github_issue: 3618 + item_type: task + title: 'Release scripts still carry 0.68.2''s V/T/MS/EPIC and AP=/mnt/... — and the hardcoded-path ratchet does not match /mnt, so nothing catches it' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:36:49Z + updated: 2026-09-21T05:36:49Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3618 "done_when"): 1. V/T/MS/EPIC are derived or passed, never literals. 2. AP is derived from the repository, not from a path under /mnt. The scripts are in-tree now; an out-of-tree bundle path is a leftover of where they used to live. 3. Either the /mnt blind spot closes (#3592) or this row ships its own assertion that no release script references a path outside the repo. Prefer the first — the blind spot affects every script, not these. 4. Falsifier: reintroduce a literal /mnt/... in a release script and require a gate to go RED. If nothing can be made to go red, item 3 is unsatisfied regardless of what was edited. SCOPE NOTE (author): this row ships its own assertion (item 3, second branch); #3592 stays open for the analyser itself.' diff --git a/docs/roadmaps/entries/PMAT-3637.yaml b/docs/roadmaps/entries/PMAT-3637.yaml new file mode 100644 index 0000000000..d1ac5714c5 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3637.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3637 + github_issue: 3604 + item_type: task + title: 'Register PMAT-3604 in the roadmap so the AD-04 quorum for PR #3634 can run (registration only; the implementation is PR #3634)' + status: planned + priority: high + assigned_to: null + created: 2026-09-20T22:39:43Z + updated: 2026-09-20T22:39:43Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:triage + notes: 'ACCEPTANCE (this row is PR #3637 itself, a roadmap-only registration): (1) docs/roadmaps/entries/PMAT-3604.yaml exists with id PMAT-3604, github_issue 3604, kind:code, and its notes transcribe issue #3604 done_when items 1-6 faithfully — every criterion present, none added, the admission rule and the out-of-scope list as the issue states them; (2) docs/roadmaps/roadmap.yaml equals aggregate(entries/) and the diff vs origin/main is additive only (deleted=0, reserialised=0); (3) the PR body carries keep-open for #3604 and the diff touches nothing outside docs/roadmaps/ except this row''s own quorum receipt(s) at docs/audits/quorum-PMAT-3637*.json. NOT in scope: any code, test or measurement of PMAT-3604 — those are PR #3634 and are judged there. A lane that finds a done_when item mis-transcribed, missing or invented FAILs this row.' diff --git a/docs/roadmaps/entries/PMAT-3644.yaml b/docs/roadmaps/entries/PMAT-3644.yaml new file mode 100644 index 0000000000..f368071f82 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3644.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3644 + github_issue: 3644 + item_type: task + title: 'check_guards_are_wired.sh reads a step NAME as a bare guard_tree.sh dispatch: 4 guards dark, incl. the #3305 publish-strip guard (which also dies on intel''s python 3.10)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T01:39:50Z + updated: 2026-09-21T01:39:50Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3644 "What lands"; minted on the PR branch so the AD-04 quorum can run). 1. check_guards_are_wired.sh drops name: lines before the invocation match, in dispatcher_wired() and the per-guard scan; rows 8-10 (the exact ci.yml step-name fixture must NOT count as a dispatch; a quoted run: line still does; a step named after a guard wires nothing); the mutation (remove the drop) turns rows 8 and 10 RED. 2. check_pathonly_devdeps_unused_in_src.sh: tomllib -> tomli -> ENV rc=2 naming interpreter and runner; the scanner ends with a SCAN-DONE trailer the shell REQUIRES; five death rows incl. the whole guard under the intel shape; comments reworded so CARGO_RE no longer classifies it out of --no-cargo, so guard-tree runs it (bare run on the tree: rc 0, 3 s). 3. unwired_guards_baseline.txt records the other three dark guards as APERTURE REVEALS (set-aperture gains a NAME-entry admission: a file the comparand carries, in a diff that changes the owning guard; six case rows, each admission beside its refusal) with the reason each may only leave; #3646 owns the pr_review pair. No workflow file is edited.' diff --git a/docs/roadmaps/entries/PMAT-3651.yaml b/docs/roadmaps/entries/PMAT-3651.yaml new file mode 100644 index 0000000000..ea37f3252f --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3651.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3651 + github_issue: 3651 + item_type: task + title: 'guard_tree.sh drops a passing guard''s output — an UNMEASURED exit-0 reads as PASS in every CI log, so #3567''s verdict can never be observed' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T05:28:51Z + updated: 2026-09-21T05:28:51Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3651 "What lands"): 1. guard_tree.sh: under a PASS row, surface every captured line whose first token is UNMEASURED or SUMMARY (indented " ~ ", capped at 5 lines + a "(+N more)" count so a chatty guard cannot flood the log). Lines where the word is not the first token are not surfaced. A guard opts a pass-summary in by printing SUMMARY …; an UNMEASURED line is surfaced whether or not the guard opted in. 2. check_fleet_pv_shapes_gate.sh: its pass row is printed as SUMMARY PASS verdict=… so the measured row reaches the log. 3. scripts/tests/guard_tree_test.sh rows: UNMEASURED surfaced; SUMMARY surfaced; the word mid-line not surfaced; a silent pass prints only its PASS row; the cap; and a mutant with the surfacing removed goes RED. NOT in scope: closing #3567 (that needs one guard-tree run on an intel runner after this lands), and the fleet pv''s own verdict.' diff --git a/docs/roadmaps/entries/PMAT-3656.yaml b/docs/roadmaps/entries/PMAT-3656.yaml new file mode 100644 index 0000000000..3b4c4f1616 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3656.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3656 + github_issue: 3656 + item_type: task + title: 'apr convert/import invent IQ-quant weights: get_tensor_f32 approximated IQ2..IQ4 from raw bytes and exited 0' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:47:55Z + updated: 2026-09-21T05:47:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3656 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). Done when: (1) get_tensor_f32 on any IQ tensor returns Err naming the tensor and its type; no path in aprender-core returns approximated IQ values; dequantize_iq_approximate is deleted. (2) apr convert … --quantize q4k and apr import on Qwen3.5-0.8B-IQ4_XS.gguf exit non-zero and write no output file. (3) A unit test fails if an approximation arm is restored (mutation: restore the 16..=23 arm → RED). Not in scope: real IQ dequantization in aprender-core (aprender-serve has dequantize_iq*, core cannot depend on serve; the route is PP-QUANT-001 #3421). Why it rides with PMAT-3601: once IQ tensors are sized, raw apr import reaches map_gguf_dtype, whose Q8_0/Q5_K refusal says "cannot represent exactly" — a GH-375 fallback trigger — so without this fix an IQ file could import invented weights where it used to be refused.' diff --git a/docs/roadmaps/entries/PMAT-3658.yaml b/docs/roadmaps/entries/PMAT-3658.yaml new file mode 100644 index 0000000000..9ed57b10ba --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3658.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3658 + github_issue: 3658 + item_type: task + title: 'Every merge-group run is tier=full: ci_test_tier.sh requires HEAD^2 but the queue is SQUASH (one parent) — a text-only PR runs ~2 runner-hours of unit tests; HEAD^1..HEAD is already the PR diff' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T06:03:14Z + updated: 2026-09-21T06:03:14Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — issue #3658 done_when as AMENDED 2026-09-21 06:10Z (the amendment supersedes the original list; operator instruction quoted there: "you will fix ASAP this is P0, and NO roadmap shouldn''t even have 10 minutes testing"). (1) A squash-shaped merge-group fixture (one parent, main moved, a leaf crate touched) gives tier=quick with that crate, not full. The same fixture with a roadmap entry only gives tier=none. Both are case-table rows. (2) Mutation: restoring the HEAD^2 gate sends the squash group back to full (a RED row). (3) A failing git diff HEAD^1..HEAD still gives tier=full (the fail-closed row is kept). (4) On a pull_request: roadmap + audit paths only -> tier=none; the same with a deleted docs path -> quick; roadmap plus any non-docs path -> quick; docs/specifications/ only -> quick. (5) A push of a squash landing (one parent, no reflog) -> quick over HEAD^1..HEAD, not full. (6) [AMENDMENT 2, 2026-09-21 06:55Z, supersedes amendment 1 item 6] ci.yml changes in exactly two places, and nothing else in the workflow changes: (a) one step, if: tier == none, that logs the reason through env:; (b) the pull_request touched-list line (git diff ... > "$RUNNER_TEMP/touched.txt" in the tier step) gains --no-renames, so both sides of every rename are touched paths. diff_into in ci_test_tier.sh gains the same flag. Rows: default --name-only on a rename fixture lists only the new path (the trap is real); the rename into docs/audits/ gives quick with the crate it left; a mutant without --no-renames gives none (RED); and a file-shaped row asserts the ci.yml line carries the flag. The other touched-list builder with the same shape (ci.yml:2783, the ci_gpu_touched feed) is pre-existing, has a different consumer, is out of scope, and is #3667. (7) After landing: the next 5 merge-group runs'' shard-1 logs show a tier= other than full for the docs-only and small-crate groups. This is observed after the merge, and the issue stays open until it is. SCOPE ADDITIONS in the amendment: A. docs-only diffs (every path under docs/roadmaps/ or docs/audits/ and present at HEAD; never a delete) and an empty diff get tier=none; B. a push landing a squash uses HEAD^1..HEAD. NOT IN SCOPE: guard-cargo and the reusable ci/* jobs on docs-only changes (the next row). Refs #3650, #3643.' diff --git a/docs/roadmaps/entries/PMAT-3662.yaml b/docs/roadmaps/entries/PMAT-3662.yaml new file mode 100644 index 0000000000..056c9e5fb6 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3662.yaml @@ -0,0 +1,23 @@ +- id: PMAT-3662 + github_issue: 3662 + item_type: task + title: 'Hand-typed ggml id→name tables outside aprender-core: sliding_window_entropy.rs, validate_manifest.rs, aprender-serve model_source.rs name 11 of 35 live types; route through trueno_quant::GgmlType' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T06:40:00Z + updated: 2026-09-21T06:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3662 done_when 1, verbatim: "Every non-example id→name mapping in `apr-cli` and `aprender-serve` resolves through `trueno_quant::GgmlType::from_id(id).map_or(, GgmlType::as_str)`, or through one shared helper that does. No `12 => "Q4_K"`-shaped match arm remains outside `aprender-quant` in those two crates'' `src/`."' + - 'Issue #3662 done_when 2, verbatim (as amended by the filer 2026-09-21 after measuring): "A test per site feeds ids 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS) and 30 (BF16) and gets the upstream names. It is RED on today''s tables where they are incomplete (sliding_window_entropy.rs, model_source.rs). validate_manifest.rs was complete when measured (amended 2026-09-21 by the filer), so its row is a parity row and passes before and after."' + - 'Issue #3662 done_when 3, verbatim: "Unknown ids (e.g. 4, 5, 999) still print the site''s existing unknown token, so output stays stable for ids ggml removed."' + - 'Issue #3662 done_when 4, verbatim: "`apr-cli` depends on the quant crate via `{ workspace = true }` (no crates.io copy)."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-7a (sliding_window_entropy.rs:295); the other two sites found by aprender-62 at origin/main c0ab699b0 with a scan of crates/apr-cli/src and crates/aprender-serve/src for ` => "Q4_K"`-shaped arms, which returns exactly three: apr-cli commands/sliding_window_entropy.rs `ggml_dtype_name` (unknown token "Unknown"; knows 11 ids), apr-cli commands/validate_manifest.rs `ggml_type_name` (unknown token "UNKNOWN"; knows ids 0-3,6-30, so it is complete today but hand-typed), aprender-serve api/model_source.rs `gguf_qtype_name` (returns None; knows 15 ids, so every IQ*, I8-I64 and F64 is None). "RED on today''s tables" is per site. validate_manifest.rs already names 10/16/23/30 correctly, so its row (ids 10/16/23/30 plus unknown ids 4/5/999, the same ids as the other two sites) is a parity row and is not expected to be RED. The two tables that are wrong today (sliding_window_entropy, model_source) carry the RED rows. validate_manifest.rs''s reverse name→id parser (`"iq4_xs" => Some(23)`) has case-insensitive aliases ("bfloat16") and is not an id→name mapping. It is out of scope and stays. Examples under crates/aprender-serve/examples/ are out of scope per the issue. + + Facts a reviewer needs, each checked at the PR head (round 1 of the quorum misread all three). (1) WHERE sliding_window_entropy.rs IS TESTED: it is not a module. crates/apr-cli/src/commands/hex.rs:442 is `include!("sliding_window_entropy.rs")`, and hex.rs:444 includes hex_06.rs, whose :11 is `include!("hex_options_detect.rs")`. So `ggml_dtype_name` and its tests share one module, `commands::hex`. The pre-existing tests for this function (test_ggml_dtype_name_known/_unknown) already live in hex_options_detect.rs, and the new test_ggml_dtype_name_names_every_family sits beside them. It ran as commands::hex::tests::test_ggml_dtype_name_names_every_family: RED on the old table (left "Unknown", right "Q2_K"), GREEN after. (2) THE CRATE NAME: the workspace dependency is `trueno-quant = { path = "crates/aprender-quant", ..., package = "aprender-quant" }` (root Cargo.toml:262), and Cargo.lock records PACKAGE names. So `aprender-quant` in apr-cli''s Cargo.lock dependency list is what cargo wrote for `trueno-quant = { workspace = true }`. It was generated by cargo, not hand-edited. (3) GgmlType IS EXPORTED: crates/aprender-quant/src/lib.rs has an unconditional `pub mod ggml_type;` and `pub use ggml_type::{GgmlFamily, GgmlType, ...}`. The type was added by #3581 AFTER the v0.68.2 tag (`git cat-file -e v0.68.2:crates/aprender-quant/src/ggml_type.rs` fails), so a tree at or before v0.68.2, or the crates.io aprender-quant 0.68.2, does not have it and fails with E0433. At the PR head, `cargo check -p aprender-serve` and `cargo check -p apr-cli` both exit 0. Verified at the PR head: cargo test -p aprender-serve --lib model_source 19 passed; cargo test -p apr-cli --lib -- commands::hex commands::validate_manifest 174 passed; clippy -D warnings clean on both crates.' diff --git a/docs/roadmaps/entries/PMAT-3670.yaml b/docs/roadmaps/entries/PMAT-3670.yaml new file mode 100644 index 0000000000..8b2a607336 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3670.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3670 + github_issue: 3670 + item_type: task + title: 'check_fleet_pv_shapes_gate.sh FAILs guard-tree on fleet state (intel pin rewritten to 0.65.2 by another repo''s CI) — fleet state must be UNMEASURED, never FAIL' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T07:33:19Z + updated: 2026-09-21T07:33:19Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — issue #3670 done_when, verbatim in substance. (1) Each fleet-state branch prints UNMEASURED runner=... reason= ..., exits 0, and never prints PASS or SUMMARY PASS. (2) A mutant restoring FAIL on pin-mismatch exits 1, and a mutant restoring the ENV exit on pv-env exits 2; each mutant row refuses to pass if its edit did not apply. (3) A planted violation from a matching pinned binary is still RED, and a control that does not fire is still RED. (4) Real runs: pin forced to 0.65.2 under a 0.68.2 binary gives UNMEASURED reason=pin-mismatch (rc 0); the fleet pv on a tree with migrated parity records gives PASS verdict=Pass. RULING: PASS/FAIL only from a verdict produced by a binary that matches its pin; fleet state is UNMEASURED with a named reason and exit 0. Scope: scripts/check_fleet_pv_shapes_gate.sh only, one commit on top of #3653 (053992448).' diff --git a/docs/roadmaps/entries/PMAT-3681.yaml b/docs/roadmaps/entries/PMAT-3681.yaml new file mode 100644 index 0000000000..74ac6189df --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3681.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3681 + github_issue: 3681 + item_type: task + title: '#3669 batch: two functions over the complexity ratchet (run_shapes_gate NEW 13/28, dispatch_runtime_commands GROWN 41->43) — extract helpers, no behaviour change' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T09:12:54Z + updated: 2026-09-21T09:12:54Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3681): 1. run_shapes_gate is under the threshold (cyclomatic < 30, cognitive <= 25) and dispatch_runtime_commands is back to main''s 15/41 — bash scripts/check_complexity_ratchet.sh reports none new, none grown. 2. No behaviour change: each extraction moves code verbatim into a helper (same order, same outcomes); cargo test -p aprender-contracts --lib passes the same count as before (1684); cargo test -p apr-cli --lib dispatch/registry pass; clippy -D warnings clean on both crates. 3. Nothing else in the diff: only shapes_gate.rs and dispatch.rs, plus this row''s roadmap fragment. CONTEXT: the base of this diff is the batch head 01abbca6d (release/0.69-batch); the functions'' prior shapes come from #3600 (run_shapes_gate) and #3604 + #3638/#3602 (dispatch_runtime_commands), which are already in the batch with their own receipts and are NOT judged here.' diff --git a/docs/roadmaps/entries/PMAT-3682.yaml b/docs/roadmaps/entries/PMAT-3682.yaml new file mode 100644 index 0000000000..a1922668b4 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3682.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3682 + github_issue: 3682 + item_type: task + title: 'parity_receipt_denominator.sh reads git''s refusal as 0 receipts on container runners — a git failure must be ENV exit 2, never a count' + status: in_progress + priority: critical + assigned_to: aprender-62 + created: 2026-09-21T09:19:00Z + updated: 2026-09-21T09:19:00Z + spec: null + acceptance_criteria: + - 'Issue #3682 done_when 1, verbatim: "The listing is `git -c safe.directory="$root" -C "$root" ls-files …`, and git''s stderr is no longer discarded."' + - 'done_when 2, verbatim: "A git failure makes the script exit **2 (ENV)** with a message naming git. It is never a count of 0. `verify` propagates that status instead of collapsing it to 1."' + - 'done_when 3, verbatim: "A self-test row puts a `git` stub on PATH that exits 128 with the dubious-ownership message. It requires exit 2 and a message naming git, and it requires that the output NOT contain \"holds 0\"."' + - 'done_when 4, verbatim: "Mutation: with the pre-fix `count_and_check` restored, that row goes RED."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'The judged diff is this ONE commit, scripts/parity_receipt_denominator.sh plus this fragment, with --base at its parent. The three commits under it on the 0.69 batch branch (the dogfood ledger re-audit, the ONT ratchet restamp, the ont4b 6->9 raise) are data/ratchet restamps that carry their proofs in their commit bodies and are outside this ticket. Found in batch #3669, workspace-test shard 3/3 on gx10-pool1 (job 106269529629): "EXPECTED_RECEIPTS says 7; the tree holds 0". On the docker runners git refuses the bind-mounted tree ("detected dubious ownership", exit 128), and the old `(cd "$root" && git ls-files … 2>/dev/null | sort -u)` inside a process substitution discarded both the message and the exit status. Measured at this commit: `bash scripts/parity_receipt_denominator.sh --self-test` 5/5 ok (the new row: "git refusing the tree is ENV exit 2 naming git, never a count of 0"); `bash scripts/parity_receipt_denominator.sh` "PASS 7 receipt(s) … says 7"; MUTANT with the pre-fix count_and_check from 55b123f2f: self-test rc=1, "FAIL git refused and the result was exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0."; `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` 9 passed. Sibling sweep, on lines the batch ADDED in scripts/ and crates/*/tests/: ci_test_tier.sh:180 and :252 fail closed (not docs-only / tier=full); lib_baseline_ratchet.sh:300/302 fail closed but misattribute the cause ("this branch WROTE it"). No other zero-count site. Outside test paths: release/lib_release_params.sh:39 silently falls back to the tree root. Those are filed, not fixed here.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 84e48c530f..e5a299237a 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -18108,86 +18108,684 @@ roadmap: estimated_effort: null labels: [] notes: null +- id: PMAT-3338 + github_issue: 3338 + item_type: task + title: 'pv proof-status --table panics on the real corpus: a byte slice lands inside a multi-byte character (#3338)' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-21T00:32:08Z + updated: 2026-09-21T00:32:08Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Transcribed from issue #3338. Repro: `pv proof-status contracts --table` on origin/main''s contracts, rc=101, panicked at crates/aprender-contracts/src/obligation_matrix.rs:169:11: "byte index 40 is not a char boundary; it is inside ''∈'' (bytes 39..42) of `for Q4_K_M Qwen2.5-Coder, quantization ∈ {Q4_K, Q6_K}`". Plain `pv proof-status` (no --table) does not panic; only the table renderer does. Cause as the issue states it: `truncate(s, max)` returns `&s[..max]`, a byte slice, which panics when `max` falls inside a multi-byte character. Done when the table renderer no longer panics on that input — truncation cuts on a char boundary, not a byte index — and `pv proof-status contracts --table` runs on the real corpus. This fix rides in PR #3351 because that PR''s L2-column change is exercised through `--table` on the real corpus, which panicked before it.' - id: PMAT-3341 github_issue: 3341 item_type: task title: MoE load-time expert-qtype contract — refuse at load, not at first token (#3341) status: planned - priority: medium + priority: medium + assigned_to: null + created: 2026-09-16T19:22:19Z + updated: 2026-09-16T19:22:19Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3346 + github_issue: 3346 + item_type: task + title: 'QE2E-INV-001: ModelConstraints dropped the gated-DeltaNet shape, so 18 of every 24 Qwen3.5 layers were counted as if their tensors did not exist (#3346)' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-20T22:54:24Z + updated: 2026-09-20T22:54:24Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE, from issue #3346 ("decide which side is wrong, with a measurement") as PR #3350 answers it — the answer is neither the range nor the descriptor but the TYPE that reads the descriptor: (1) ModelConstraints carries inner_size, state_size, conv_kernel, group_count and full_attention_interval from contracts/model-families/qwen3_5.yaml, which it previously dropped; (2) the config-derived model_parameter_count equals the measured GGUF tensor sum of a REAL file — ~/models/Qwen3.5-0.8B-Q4_K_M.gguf, 320 tensors, 752,393,024 parameters, delta 0 — with the gated-DeltaNet layer (21,555,360) and the full-attention layer (18,352,640) each matching the measured blk; (3) the two shapes that contradict dense accounting are modelled from the tensors, not assumed: attn_q is [d, 2*n_h*d_k] (the q projection emits the output gate) and the DeltaNet mixer projections/conv/state norm are counted for the 18-of-24 linear layers; (4) the 9B range instantiation is NOT asserted in this PR — the repo 9b descriptor disagrees with itself (inner_size 2048 at hidden 4096; group_count 8 fails group_count*state_size == inner_size) so P(9B) computes to 8.345B, and settling it needs a real Qwen3.5-9B GGUF, recorded in the QE2E-INV-001 binding note; (5) the note edit leaves contracts/binding.yaml valid YAML (156 entries, same as main) and the tracked contracts.nt equals a fresh extraction. Out of scope: acquiring the 9B GGUF; widening the range.' +- id: PMAT-3347 + github_issue: 3347 + item_type: task + title: bind the six qwen35-e2e equations to implementations + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T06:52:28Z + updated: 2026-09-16T06:52:28Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3351 + github_issue: 3347 + item_type: task + title: pv's L2 column reads an obligation-to-test link, not an index + status: in_progress + priority: medium + assigned_to: null + created: 2026-09-16 07:53:27+00:00 + updated: 2026-09-16 07:53:27+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE, transcribed from issue #3347 "Ask" — this row is the second and third bullets; the first bullet (write the six implementation bindings) was #3348 / roadmap PMAT-3347 and is out of scope here. Verbatim from the issue: (2) "Make the --table L2 column read the actual obligation<->test link rather than an index comparison" — the issue states the defect as: obligation_matrix.rs computes l2_tested as idx < falsification_tests.len(), so obligations showed an L2 tick before any test existed; a column that ticks on a count, not on a link, cannot report coverage. (3) "Make single-file pv lint --strict-test-binding resolve its scan root to the repo, or refuse the single-file form instead of reporting a false negative" — the issue states the defect as: it sets the scan root to the contract''s parent, contracts/, which contains no crates/, so no ref can resolve and an already-bound control contract fails identically. Either resolution named in the issue satisfies (3). No count of obligations, tests or contracts is a criterion.' +- id: PMAT-3359 + github_issue: 3359 + item_type: task + title: H1 SIMD dot speedup assertion is a wall-clock check in a required gate + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:10:03Z + updated: 2026-09-16T12:10:03Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3365 + github_issue: 3365 + item_type: task + title: apr install.sh — curl one-liner installer with path-scoped CI test (#3365) + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T15:03:48Z + updated: 2026-09-16T15:03:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3367 + github_issue: 3367 + item_type: task + title: apr chat exits 0 after a generate failure (#3367) + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T14:53:17Z + updated: 2026-09-16T14:53:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3370 + github_issue: 3370 + item_type: task + title: 'EPIC: CRUX category O — AutoML parity vs AutoGluon 1.6.3 — 9 P0, 9 P1, 6 P2 (24 stories)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: docs/specifications/crux-competitive-research-ux-workflows.md + acceptance_criteria: + - 'All 24 children (#3371-#3394) closed or explicitly re-cut; apr automl fit/predict on iris and apr forecast fit on a 5-item panel run end to end; each child lands with a contract and mutation-RED evidence.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - epic + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3371 + github_issue: 3371 + item_type: task + title: 'P0: One-call tabular AutoML: fit(label) -> predict on a CSV — CRUX-O-01' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-01, competitor autogluon, demand 5. No predictor-level AutoML entry point exists. crates/aprender-core/src/automl/ is a hyperparameter TUNER (AutoTuner, TPE, GridSearch, RandomSearch, DESearch, TimeBudget) that tunes ONE estimator the caller already chose; nothing takes a labelled table and returns a fitted model. `apr train` is causal-LM pre-training only (crates/apr-cli/src/commands/train.rs:1-5) and `apr finetune --task classify` is text classification. FALSIFIABLE (contracts/crux-O-01-v1.yaml): fit on a labelled CSV returns a predictor whose predictions score above the majority-class baseline — on the iris fixture (crates/aprender-core/src/datasets/iris.csv) accuracy >= 0.90 AND the majority-class baseline is asserted at 0.333 in the same test; the same call works for a regression label without a problem_type argument — a numeric label column yields a regressor whose R^2 on a held-out split exceeds 0.5 AND the mean-predictor baseline is asserted at ~0.0; apr automl fit is reachable from the CLI — `apr automl fit --label class train.csv --out model.apr` exits 0 and writes a loadable artifact; a missing --label exits 2 with a message naming the flag. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3372 + github_issue: 3372 + item_type: task + title: 'P0: Problem-type inference: binary / multiclass / regression / quantile from the label column — CRUX-O-02' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-02, competitor autogluon, demand 5. No function infers a task from a label column. Estimators are chosen by type name (LogisticRegression vs LinearRegression); DataFrame in crates/aprender-core/src/data/mod.rs carries ColumnStats but no label-kind classifier. FALSIFIABLE (contracts/crux-O-02-v1.yaml): two unique label values infer binary, 3..=N small-cardinality infer multiclass, many-unique numeric infers regression — a table of (label column, expected kind) fixtures including the ambiguous cases {0,1} as int, {0.0,1.0} as float, and 30 unique floats over 1000 rows all classify as documented; the override wins and a contradictory override is rejected — problem_type=regression on a string label column is an Err naming the column, not a silent cast. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3373 + github_issue: 3373 + item_type: task + title: 'P0: Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget — CRUX-O-03' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-03, competitor autogluon, demand 5. No preset vocabulary. AutoTuner takes a SearchSpace the caller hand-builds (crates/aprender-core/src/automl/params.rs); there is no named bundle of {models, bagging, stacking, time_limit}. FALSIFIABLE (contracts/crux-O-03-v1.yaml): every preset name resolves to a portfolio and unknown names are rejected with the valid list — the five quality presets each yield a non-empty ordered model list; `presets="bestest"` is an Err whose message contains all five valid names; presets are ordered: a higher preset never fits FEWER model families than the one below it — for medium < good < high < best the family count is monotone non-decreasing, asserted pairwise. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3374 + github_issue: 3374 + item_type: task + title: 'P0: Leaderboard: per-model validation/test score, fit time, predict time and stack level — CRUX-O-04' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-04, competitor autogluon, demand 5. No leaderboard type. GridSearchCVResult in model_selection/ ranks parameter settings of one estimator; TuneResult in automl/tuner.rs is a single best trial. Nothing tabulates several fitted models with timings. FALSIFIABLE (contracts/crux-O-04-v1.yaml): the leaderboard is sorted by validation score descending and its columns are fixed — columns == [model, score_val, score_test?, pred_time_val, fit_time, stack_level, fit_order] and score_val is non-increasing row to row; timings are measured, not defaulted — every fit_time and pred_time_val is > 0 after a real fit; a leaderboard built with no fit has zero rows, not zero timings; apr automl leaderboard prints the same table from a saved artifact — `apr automl leaderboard model.apr --json` emits the rows byte-equal to the in-process leaderboard. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3375 + github_issue: 3375 + item_type: task + title: 'P0: Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) — CRUX-O-05' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-05, competitor autogluon, demand 5. Encoders exist (LabelEncoder, OneHotEncoder, OrdinalEncoder, StandardScaler, PolynomialFeatures in crates/aprender-core/src/preprocessing/) but every one is applied by hand to a column the caller already typed. There is no pass that reads a raw DataFrame, infers each column''s kind, and emits a fitted transform. Datetime expansion and text n-gram features do not exist. TfidfVectorizer exists in text/ but is not wired to a tabular pipeline. FALSIFIABLE (contracts/crux-O-05-v1.yaml): column kinds are inferred from raw values — a fixture CSV with int, float, low-cardinality string, high-cardinality string, ISO datetime and free-text columns is typed as {numeric, numeric, categorical, text, datetime, text} exactly; constant and duplicate columns are dropped and the drop is reported — a column with one unique value and an exact duplicate of another column are both absent from the transformed output AND named in the fit report; the fitted pipeline is deterministic under transform — transform(train) then transform(train) are byte-identical and transform(test) never sees a category unseen at fit as anything but the reserved unknown code. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3376 + github_issue: 3376 + item_type: task + title: 'P1: K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) — CRUX-O-06' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-06, competitor autogluon, demand 4. KFold and StratifiedKFold exist (model_selection/) and cross_validate scores them, but no wrapper trains one child per fold, keeps all children, averages them at predict time and exposes the out-of-fold matrix. RandomForest bags trees internally and is not reusable for other estimators. FALSIFIABLE (contracts/crux-O-06-v1.yaml): OOF predictions cover every training row exactly once — for n rows and k folds the OOF matrix has n rows, no NaN, and each row was predicted by the one child that did not see it (asserted through a fold-id trace); bagged prediction is the mean of the children — predict_proba of the bag equals the elementwise mean of the k children''s predict_proba within 1e-12. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3377 + github_issue: 3377 + item_type: task + title: 'P1: Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) — CRUX-O-07' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-07, competitor autogluon, demand 4. No stacking. crates/aprender-core/src/stack/ is a deployment-health module (StackHealth, InferenceConfig), not a model stacker. ensemble/ holds MixtureOfExperts gating only. FALSIFIABLE (contracts/crux-O-07-v1.yaml): level-2 features are OOF, never in-sample — a mutation that feeds in-sample level-1 predictions to level 2 is detected by the leak test: the L2 holdout score on a pure-noise label rises above chance (asserted RED) while the OOF path stays at chance; dynamic stacking falls back when stacking hurts — on a fixture where L2 holdout score < L1 holdout score, the final model is the L1 ensemble and the decision is recorded in the fit summary. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3378 + github_issue: 3378 + item_type: task + title: 'P0: Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) — CRUX-O-08' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-08, competitor autogluon, demand 5. No ensemble-selection algorithm. MixtureOfExperts learns a gating network (ensemble/moe.rs); nothing performs the forward greedy selection with replacement over base-model validation predictions that yields non-negative weights summing to 1. FALSIFIABLE (contracts/crux-O-08-v1.yaml): weights are a probability vector and the ensemble never scores below its best member — sum(w)=1, all w>=0, and validation metric(ensemble) >= max over members within 1e-9, on 3 fixtures; selection is greedy with replacement and reproducible — with ensemble_size=25 the weight of a member equals its selection count / 25; two runs on the same inputs produce identical weights. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3379 + github_issue: 3379 + item_type: task + title: 'P0: Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share — CRUX-O-09' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-09, competitor autogluon, demand 5. TimeBudget and EarlyStopping exist in automl/tuner.rs but budget ONE tuner. There is no allocation of a global limit across an ordered portfolio, no per-model time share, and no ''skip the rest'' when the budget is exhausted. FALSIFIABLE (contracts/crux-O-09-v1.yaml): the wall-clock of fit never exceeds time_limit by more than the tolerance — with time_limit=5s on a portfolio that would take >60s unconstrained, elapsed <= 5s + 1s AND at least one model was skipped with reason=budget in the summary; the budget is redistributed when a model finishes early — a model that uses 10% of its share returns the remainder to the pool; the next model''s share is asserted larger than the naive equal split. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3380 + github_issue: 3380 + item_type: task + title: 'P1: Predictor-level permutation feature importance with p-values and confidence intervals — CRUX-O-10' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-10, competitor autogluon, demand 4. PermutationImportance exists in crates/aprender-core/src/interpret/ for a single estimator. Missing: the predictor-level call on raw (pre-pipeline) columns, num_shuffle_sets repeats, stddev / p-value / p99 columns, and subsampling. AutoGluon 1.6 also cut this call''s memory 25x (#5645). FALSIFIABLE (contracts/crux-O-10-v1.yaml): importance is reported per raw input column — on a fixture whose datetime column expands to 4 features, the importance table has one row for the datetime column, not four; a pure-noise column has importance statistically indistinguishable from zero — with num_shuffle_sets=10 the noise column''s p-value > 0.05 AND the signal column''s p-value < 0.01 on the same run. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3381 + github_issue: 3381 + item_type: task + title: 'P1: Decision-threshold calibration for binary metrics (calibrate_decision_threshold) — CRUX-O-11' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-11, competitor autogluon, demand 4. Probability calibration exists (PlattScaling, IsotonicRegression, TemperatureScaling in calibration.rs) but nothing searches the decision threshold that maximises f1 / balanced_accuracy / mcc on validation data and stores it on the predictor (`decision_threshold`, `set_decision_threshold`). FALSIFIABLE (contracts/crux-O-11-v1.yaml): the calibrated threshold beats 0.5 on the calibration metric and the gain is asserted, not the search — on an imbalanced fixture (5% positives) f1 at the calibrated threshold > f1 at 0.5 by >= 0.05 absolute; the threshold is persisted with the model — save then load reproduces predict() bit-identically including the threshold; a mutation that resets the threshold to 0.5 on load turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3382 + github_issue: 3382 + item_type: task + title: 'P2: refit_full: retrain the selected models on train+validation after model selection — CRUX-O-12' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-12, competitor autogluon, demand 3. No refit-on-full-data step. cross_validate and grid_search return scores; the model returned is the one fitted on a fold, not on all rows. FALSIFIABLE (contracts/crux-O-12-v1.yaml): the refit model saw every row — the refit estimator''s training-row count equals n_train + n_val, asserted through the fitted row count it reports; refit keeps the selected hyperparameters and drops the bag children — the artifact after refit has one child per selected model and its hyperparameters are byte-equal to the pre-refit winner''s. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3383 + github_issue: 3383 + item_type: task + title: 'P2: Model distillation: compress the ensemble into one fast student — CRUX-O-13' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-13, competitor autogluon, demand 3. Distillation exists only for LLMs in the training crate (knowledge distillation of transformers); nothing trains a single tabular student on the teacher ensemble''s soft labels with data augmentation. FALSIFIABLE (contracts/crux-O-13-v1.yaml): the student is faster and within tolerance of the teacher — student pred_time < 0.25 x teacher pred_time AND student validation score >= teacher score - 0.02 on a fixture; augmentation produces rows the training set does not contain — with augment_method=spunge the student''s training set has > n_train rows and the added rows are not row-equal to any original. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3384 + github_issue: 3384 + item_type: task + title: 'P1: Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file — CRUX-O-14' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-14, competitor autogluon, demand 4. Single estimators serialize to .apr (bundle/, serialization/). No artifact holds a fitted feature pipeline (O-05) plus several models plus ensemble weights plus a decision threshold and loads them as one predictor. Nothing prunes non-selected models from disk. FALSIFIABLE (contracts/crux-O-14-v1.yaml): a deployment clone is smaller and predicts identically — clone_for_deployment artifact bytes < 0.5 x full artifact bytes AND predict() on the clone is bit-identical to predict() on the original for the test fixture; the artifact is self-describing and refuses a schema drift — load() on an artifact whose feature schema disagrees with the input columns is an Err naming the first mismatched column, not a wrong prediction. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3385 + github_issue: 3385 + item_type: task + title: 'P2: Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size) — CRUX-O-15' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-15, competitor autogluon, demand 3. No model or ensemble is ever excluded for being slow at predict time; leaderboard timings (O-04) do not exist to compare against. FALSIFIABLE (contracts/crux-O-15-v1.yaml): the selected ensemble respects the per-row latency limit — with infer_limit=L the measured pred_time per row of the final model <= L AND at least one faster-but-worse model was preferred over a slower-but-better one (asserted on a fixture built to force the trade). Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3386 + github_issue: 3386 + item_type: task + title: 'P2: Fit diagnostics: fit_summary, model_failures and learning curves — CRUX-O-16' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-16, competitor autogluon, demand 3. ProgressCallback in automl/tuner.rs streams trial results; there is no post-fit summary object listing models trained, models failed with their error, per-model hyperparameters and per-iteration validation curves. FALSIFIABLE (contracts/crux-O-16-v1.yaml): a model that raises during fit is recorded, not swallowed and not fatal — a portfolio containing a deliberately failing model finishes; model_failures() has exactly one row naming the model and the error string; the leaderboard omits it; learning curves have one point per boosting/epoch iteration — for a GBM with n_estimators=50 and learning_curves=true the curve has 50 validation points, monotone non-increasing after early-stopping''s best iteration is asserted absent. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3387 + github_issue: 3387 + item_type: task + title: 'P1: Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) — CRUX-O-17' + status: planned + priority: high + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-17, competitor autogluon, demand 4. No tabular in-context learner. nn/ and the inference crates run causal LMs; nothing consumes (X_train, y_train, X_test) as one context and predicts without gradient steps. AutoGluon 1.4-1.6 added TabPFNv2/2.5/2.6/3, TabICL/v2, Mitra, TabDPT, Nori and made them the extreme preset. FALSIFIABLE (contracts/crux-O-17-v1.yaml): zero-gradient prediction matches a pinned reference within tolerance — loading a pinned small checkpoint via apr pull and predicting a 100-row fixture matches the committed reference probabilities within 1e-4; the context limit is enforced, not silently truncated — a context of rows > the model''s documented max is an Err naming the limit; a mutation that truncates instead turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P1 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3388 + github_issue: 3388 + item_type: task + title: 'P2: Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit — CRUX-O-18' + status: planned + priority: medium + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-18, competitor autogluon, demand 3. No estimator reports an expected peak memory before fitting and nothing checks a limit. AutoGluon 1.6 spent 4 PRs on this (#5757, #5768, #5791, GPU budgeting for parallel folds). FALSIFIABLE (contracts/crux-O-18-v1.yaml): a model whose estimate exceeds the limit is skipped with reason=memory — with memory_limit=64MiB on a fixture where the estimate for the largest model is > 64MiB, that model is absent from the leaderboard and present in the skip list with its estimate; the estimate is not a constant — the estimate for a 10x larger fixture is asserted larger than for the base fixture; a mutation returning a constant turns this RED. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P2 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3389 + github_issue: 3389 + item_type: task + title: 'P0: Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq — CRUX-O-19' + status: planned + priority: critical + assigned_to: null + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z + spec: null + acceptance_criteria: + - 'CRUX-O-19, competitor autogluon, demand 5. crates/aprender-core/src/time_series/mod.rs is one struct, ARIMA, on one f32 series (fit/forecast/order). There is no panel container keyed by item and timestamp, no frequency, no horizon-first API and no per-item forecast. FALSIFIABLE (contracts/crux-O-19-v1.yaml): a panel of N items forecasts N x prediction_length rows — on a fixture of 5 items with 200 hourly points each and prediction_length=24 the output has exactly 120 rows, each (item_id, timestamp) unique, timestamps continuing each item''s last stamp at freq=h; irregular timestamps are rejected or regularised, never silently misaligned — an item with a missing hour is an Err naming the item and gap unless fill=forward is passed, in which case the filled row is flagged. Epic #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: null +- id: PMAT-3390 + github_issue: 3390 + item_type: task + title: 'P0: Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) — CRUX-O-20' + status: planned + priority: critical assigned_to: null - created: 2026-09-16T19:22:19Z - updated: 2026-09-16T19:22:19Z + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z spec: null - acceptance_criteria: [] + acceptance_criteria: + - 'CRUX-O-20, competitor autogluon, demand 5. ARIMA.forecast returns a point path; metrics/probabilistic.rs holds classification-probability metrics, and metrics/regression.rs has no seasonal-scaled (MASE/RMSSE) or quantile (WQL/MQL) losses. FALSIFIABLE (contracts/crux-O-20-v1.yaml): quantile forecasts are monotone in the quantile level — for every (item, step) q0.1 <= q0.5 <= q0.9, asserted over the whole fixture; a mutation that shuffles the quantile columns turns this RED; each metric matches the AutoGluon reference value on a pinned fixture — WQL, MASE, SMAPE, RMSSE, WAPE and MQL computed on a committed (y_true, y_pred, quantiles) fixture equal the reference values produced by autogluon.timeseries.metrics within 1e-9. Epic #3370.' phases: [] subtasks: [] estimated_effort: null - labels: [] + labels: + - P0 + - enhancement + - crux + - pareto-autogluon notes: null -- id: PMAT-3347 - github_issue: 3347 +- id: PMAT-3391 + github_issue: 3391 item_type: task - title: bind the six qwen35-e2e equations to implementations + title: 'P1: Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions — CRUX-O-21' status: planned - priority: medium + priority: high assigned_to: null - created: 2026-09-16T06:52:28Z - updated: 2026-09-16T06:52:28Z + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z spec: null - acceptance_criteria: [] + acceptance_criteria: + - 'CRUX-O-21, competitor autogluon, demand 4. No time-aware validation. KFold shuffles rows; nothing cuts the last k horizons of each item as expanding-window validation sets, and there is no API that returns the validation forecasts for inspection. FALSIFIABLE (contracts/crux-O-21-v1.yaml): validation windows never contain a timestamp later than the training cut for that window — for num_val_windows=3 and val_step_size=h, each window''s max training timestamp < min validation timestamp per item, asserted for every item and window; backtest_predictions rows align with backtest_targets — the two frames have identical (window, item_id, timestamp) index sets and no NaN targets. Epic #3370.' phases: [] subtasks: [] estimated_effort: null - labels: [] + labels: + - P1 + - enhancement + - crux + - pareto-autogluon notes: null -- id: PMAT-3359 - github_issue: 3359 +- id: PMAT-3392 + github_issue: 3392 item_type: task - title: H1 SIMD dot speedup assertion is a wall-clock check in a required gate + title: 'P2: Known covariates, past covariates and static features in forecasting — CRUX-O-22' status: planned priority: medium assigned_to: null - created: 2026-09-16T12:10:03Z - updated: 2026-09-16T12:10:03Z + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z spec: null - acceptance_criteria: [] + acceptance_criteria: + - 'CRUX-O-22, competitor autogluon, demand 3. ARIMA is univariate. No container carries per-item static features or time-varying covariates, and no model consumes them. FALSIFIABLE (contracts/crux-O-22-v1.yaml): a known covariate that fully determines the target is used — on a fixture where y = 10*holiday + noise, a model given known_covariates has MASE < 0.5 x the same model without them; future covariates missing for the horizon is an error — predict() without known_covariates for all prediction_length steps is an Err naming the first missing timestamp. Epic #3370.' phases: [] subtasks: [] estimated_effort: null - labels: [] + labels: + - P2 + - enhancement + - crux + - pareto-autogluon notes: null -- id: PMAT-3365 - github_issue: 3365 +- id: PMAT-3393 + github_issue: 3393 item_type: task - title: apr install.sh — curl one-liner installer with path-scoped CI test (#3365) + title: 'P1: Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston — CRUX-O-23' status: planned - priority: medium + priority: high assigned_to: null - created: 2026-09-16T15:03:48Z - updated: 2026-09-16T15:03:48Z + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z spec: null - acceptance_criteria: [] + acceptance_criteria: + - 'CRUX-O-23, competitor autogluon, demand 4. ARIMA exists (fixed order; no auto-order search). SeasonalNaive, ETS, Theta and the intermittent-demand family (Croston, ADIDA, IMAPA) are absent. AutoGluon runs these as its ''local'' tier (timeseries/models/local/) and they anchor every leaderboard. FALSIFIABLE (contracts/crux-O-23-v1.yaml): SeasonalNaive is exactly the lag-m copy — forecast[t] == y[t - m] for the whole horizon on a fixture with m=24, byte-equal; AutoARIMA selects the planted order — on a fixture generated from ARIMA(2,1,1) the selected (p,d,q) equals (2,1,1) in >= 9 of 10 seeds; ETS and Theta match the statsforecast reference — point forecasts on the AirPassengers fixture equal the statsforecast reference within 1e-6. Epic #3370.' phases: [] subtasks: [] estimated_effort: null - labels: [] + labels: + - P1 + - enhancement + - crux + - pareto-autogluon notes: null -- id: PMAT-3367 - github_issue: 3367 +- id: PMAT-3394 + github_issue: 3394 item_type: task - title: apr chat exits 0 after a generate failure (#3367) + title: 'P1: Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning — CRUX-O-24' status: planned - priority: medium + priority: high assigned_to: null - created: 2026-09-16T14:53:17Z - updated: 2026-09-16T14:53:17Z + created: 2026-09-16T12:00:00Z + updated: 2026-09-16T12:00:00Z spec: null - acceptance_criteria: [] + acceptance_criteria: + - 'CRUX-O-24, competitor autogluon, demand 4. apr pull fetches LLM checkpoints (Category A) and the inference crates run decoder transformers, but no forecasting head exists: nothing tokenises a numeric series into the model''s input, samples a horizon, and maps it back to quantiles. FALSIFIABLE (contracts/crux-O-24-v1.yaml): zero-shot output matches a pinned reference — a pinned small checkpoint pulled by `apr pull` forecasts the committed fixture within 1e-3 of the reference quantiles; fine-tuning changes the weights and improves the in-domain metric — after fine_tune on the fixture the WQL improves by >= 5% relative AND at least one weight tensor differs from the pulled checkpoint. Epic #3370.' phases: [] subtasks: [] estimated_effort: null - labels: [] + labels: + - P1 + - enhancement + - crux + - pareto-autogluon notes: null +- id: PMAT-3401 + github_issue: 3401 + item_type: task + title: 'Admit AutoGluon 1.6.3 to CRUX as category O: 24 DRAFT contracts crux-O-01..24 with LIVE-PENDING gates (no implementation, no CLI, no tests), CRUX_COMPETITORS 14->15 in validator.rs + both crux_intake_tests.rs lists, CATEGORY_NAMES[O] in scripts/crux_scaffold_contracts.py, 24 master-registry rows + evidence_sources.autogluon, spec v2.3 §3/§5/§6, evidence/crux/autogluon/ (4 files), 26 roadmap fragments (PMAT-3370 epic, PMAT-3371..3394 stories, PMAT-3401 this ticket) + regenerated roadmap.yaml, and the three corpus derivatives (census, contracts.nt, README CONTRACT_COUNT) regenerated to the merged tree — child of #3370' + status: in_progress + priority: critical + assigned_to: null + created: 2026-09-16T16:30:00Z + updated: 2026-09-16T16:30:00Z + spec: docs/specifications/crux-competitive-research-ux-workflows.md + acceptance_criteria: + - 'Exactly this inventory, nothing more: contracts/crux-O-01..24-v1.yaml (24 files, registry: false, every gate LIVE-PENDING); crates/aprender-contracts/src/schema/validator.rs (CRUX_COMPETITORS 14->15, + autogluon with rationale) and crux_intake_tests.rs (autogluon in the corpus-vocabulary list and in the not-a-BEAT-pillar list); scripts/crux_scaffold_contracts.py (CATEGORY_NAMES O); contracts/crux-competitive-research-ux-v1.yaml (24 stories rows, coverage_intake 43/77/171/291, evidence_sources.autogluon, autogluon reference); docs/specifications/crux-competitive-research-ux-workflows.md (v2.3 header, §3 rows for N and O, §5 Category O table, §6 note); evidence/crux/autogluon/{README.md,api-surface.md,hello.sh,readme-verbs.txt}; docs/roadmaps/entries/PMAT-3370.yaml + PMAT-3371..3394.yaml + PMAT-3401.yaml (26 fragments) and docs/roadmaps/roadmap.yaml regenerated by the aggregator; README.md CONTRACT_COUNT 1842->1866. Absent on purpose: library code, apr CLI wiring, test bodies — those are the 24 children of #3370.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + - enhancement + - crux + - pareto-autogluon + notes: 'ACCEPTANCE, stated as invariants and not as counts (the first version of this row pinned "README CONTRACT_COUNT 1842->1866" and "coverage_intake 267->291" from the 2026-09-16 tree; main moved and the numbers described a tree that no longer existed, which is what the AD-04 quorum on #3395 round 0 correctly refused): (1) 24 new contract files contracts/crux-O-01-v1.yaml .. crux-O-24-v1.yaml, each metadata.category O, competitor AutoGluon, status draft, intake_status missing, registry false, and each passing pv validate; (2) CRUX_COMPETITORS grows by exactly one entry (autogluon) in crates/aprender-contracts/src/schema/validator.rs and in both crux_intake_tests.rs lists; (3) CATEGORY_NAMES gains O in scripts/crux_scaffold_contracts.py; (4) the master registry gains one row per new contract and evidence_sources.autogluon; (5) 26 roadmap fragments exist (PMAT-3370, PMAT-3371..PMAT-3394, PMAT-3401) and docs/roadmaps/roadmap.yaml equals aggregate(entries/), additive vs origin/main (deleted=0, reserialised=0); (6) contracts/census.json n_files equals origin/main''s n_files at merge time plus 24, contracts/contracts.nt equals a fresh pv extract of the merged tree, and README''s CONTRACT_COUNT blocks equal census n_files — the three derivatives ride in this PR because adding a contract stales all three (#3569). Known and NOT in scope: the census files these 24 under kind kernel because they carry no metadata.kind and registry:false (main''s crux contracts carry registry:true and file under registry); a draft/intake bucket for the census is a census question (#3640-class), not this PR''s.' - id: PMAT-3427 github_issue: 3427 item_type: task @@ -18628,6 +19226,39 @@ roadmap: estimated_effort: null labels: [] notes: null +- id: PMAT-3495 + github_issue: 3495 + item_type: task + title: 'VERIFY-001: run the 132 Kani harnesses in CI (proof credit from runs, not declarations) and pilot Verus on one dequant/parser function' + status: planned + priority: medium + assigned_to: null + created: 2026-09-18T16:43:48Z + updated: 2026-09-18T16:43:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-3496 + github_issue: 3495 + item_type: task + title: 'Register PMAT-3495 (VERIFY-001 on 0.71.0) in the roadmap so the AD-04 quorum for PR #3496 can run (registration only)' + status: planned + priority: medium + assigned_to: null + created: 2026-09-21T01:39:59Z + updated: 2026-09-21T01:39:59Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:triage + notes: 'ACCEPTANCE (this row is PR #3496 itself, a roadmap-only registration): (1) docs/roadmaps/entries/PMAT-3495.yaml exists with id PMAT-3495, github_issue 3495, and a title that transcribes issue #3495''s title VERBATIM, character for character: "VERIFY-001: run the 132 Kani harnesses in CI (proof credit from runs, not declarations) and pilot Verus on one dequant/parser function" (read from `gh issue view 3495 --json title` on 2026-09-21; the issue has no rename events); (2) docs/roadmaps/roadmap.yaml equals aggregate(entries/) and the diff vs origin/main is additive only (deleted=0, reserialised=0) after the merges of origin/main; (3) the PR touches nothing outside docs/roadmaps/ and docs/audits/. NOT in scope: any Kani harness, CI wiring or proof — those are PMAT-3495''s own work and are judged when it lands. A lane that finds the entry mis-transcribed, missing or invented FAILs this row.' - id: PMAT-3500 github_issue: 3500 item_type: task @@ -18955,7 +19586,7 @@ roadmap: estimated_effort: null labels: - kind:code - notes: 'ACCEPTANCE (hand-entered; pmat work add derived neither spec: nor acceptance_criteria: — defect paiml-mcp-agent-toolkit#1414). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). Done when: contracts/parity-receipt-v1.yaml with its shape on origin/main; extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; ONT-4c3 bound with the parity receipt as focus node; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml.' + notes: 'ACCEPTANCE (transcribed from gh#3577, incl. the dated AMENDMENT of 2026-09-21 02:00Z which supersedes the original item 1; §6 1e — the fragment transcribes the issue, it does not edit it). Spec: docs/specifications/ruling-receipts-under-contract.md (operator ruling 2026-09-20). ITEM 1 AS AMENDED, verbatim: "contracts/parity-receipt-v2.yaml carries the shape (the constraints table below, unchanged) in its shape: block, exported to contracts/shapes.ttl. contracts/parity-receipt-v1.yaml is the SHAPELESS superseded form and says why in its own text: no v1 instance exists after the back-fill, and a shape over zero instances is a vacuous pass. A v1 record is refused BY NAME by the extractor (extract:parity-receipt) and, independently, by scripts/parity_receipt_denominator.sh — two readers, both wired (#3600 ci.yml step). A future v1 instance is therefore a refusal, not a silent pass." WHY AMENDED: the original item 1 said "parity-receipt-v1.yaml with its shape" and predates the migration — all 7 records went to v2 in the same commit, so v1 has ZERO instances, and a shape over a class nothing instantiates passes vacuously, which is the defect #3610 names and #3622 makes the shapes gate DECLINE on. Following it literally would add in one PR the thing the other PR exists to refuse. Both pro lanes read the original correctly and failed the PR on it; the ruling is on the acceptance, not the lanes. ONT-4c3 LEDGER BINDING: that ledger lives in paiml/infra (ONT-001 v4.10, infra#831) and CANNOT be done from this repository. The aprender-side deliverable is extractor + shape + denominator + receipts; the ledger row is infra s. The acceptance reads "bound in the ONT-001 ledger" as infra s row, CITED, not as a change here. REMAINING done_when, unchanged: extract:parity-receipt implemented; pv lint --gate shapes --path evidence/ in the required check; PR body shows 7 older receipts RED before back-fill and 0 after, plant violation = 1, mutation RED, pc_shape fired; check_parity_receipt.sh folded into the shape or its remainder listed under not_expressible; verdict rendered on a fleet host once the pv pin lands, else checkout-only: true. Sigma parent bound: json. Back-fill denominator: 7. STOP: subset-insufficient; shared-file-touched without the guard label; any threshold typed into the shape instead of resolved from thresholds.yaml; and do NOT add a v1 shape — see the amendment.' - id: PMAT-3587 github_issue: 3587 item_type: task @@ -18973,6 +19604,91 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from issue #3587 "What lands"; minted on the PR branch so the AD-04 quorum can run -- pmat work status refuses a missing item). Two halves: the explicit-command runner (#3616, merged) and the nextest [profile.ci] key (#3626). Lands: 1. **--no-fail-fast on the shard.** The cost is runner minutes on an already-red shard; the benefit is that a red run still measures everything else it was going to measure — which is exactly when you most want the data. 2. **The reducer reports ran / skipped / total** in the shard receipt. The skipped count is a **number**, never an absence. 3. **A receipt with skipped > 0 is Unknown, not Fail and not Pass** — it names what was not measured, per doctrine 4. ' +- id: PMAT-3601 + github_issue: 3601 + item_type: task + title: 'apr inspect has no byte-size entry for any IQ-family ggml type — aprender-core sizes and names ggml types from trueno_quant::TRAITS' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:47:55Z + updated: 2026-09-21T05:47:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3601 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). Done when: (1) apr inspect succeeds on a GGUF containing any of the 10 listed ids (repro: Qwen3.5-4B-UD-Q4_K_XL.gguf, dtype 23). (2) A regression test pins all 10 ids against shape.rs''s byte_size, analogous to ggml_type_table.rs''s spot_checks_against_ggml_sizes, so a future edit can''t silently drop one back out. (3) The error message, if a truly unknown id is ever hit, names the id and points at contracts/apr-cli-commands-v1.yaml''s inspect docs or similar — not a bare "Unsupported dtype N". DEVIATION FROM THE ISSUE''S FIX PROSE (not its done_when): the issue proposed porting rows from aprender-serve''s GGML_TYPES because "the unified TRAITS crate #3430 proposes doesn''t exist yet"; #3581 has since landed it (aprender-quant ggml_type.rs TRAITS[43], upstream-extracted, fixture-checked) and aprender-core already depends on it, so shape.rs reads TRAITS instead of becoming a fifth hand copy. ALSO IN THIS DIFF (same defect class, measured on apr 0.68.2 00e5d2dde): (a) apr tensors sized tensors from a second hand-typed table in aprender-core (safetensors.rs ggml_dtype_element_size SIZES) with seven IQ rows wrong and the 26-30 tail misordered (BF16 0.375 bytes/elem) — it refused the valid Qwen3.5-0.8B-UD-IQ2_XXS.gguf as "Truncated GGUF … 167936 bytes too short"; now type_size/blck_size from TRAITS. (b) ggml_dtype_name (a third copy, ids 0-30) reads TRAITS so TQ1_0/TQ2_0 are named rather than "unknown"; contracts/apr-inspect-dtype-naming-v1.yaml GGML_NAMES + KANI bound amended to match. Behaviour preservation: every id sized before keeps its exact byte count (floor arithmetic kept; characterization test over non-block-multiple counts), ids 0-30 keep their names. The IQ fabrication this sizing exposes is PMAT-3656 (#3656), fixed in the same diff.' +- id: PMAT-3602 + github_issue: 3602 + item_type: task + title: 'apr run --gpu reported a CPU fallback as success: registry::after_generation had no production caller' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-21T00:00:00Z + updated: 2026-09-21T00:00:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3602 done_when; pmat work add derives neither spec: nor acceptance_criteria: — paiml-mcp-agent-toolkit#1414). Item 1: the GPU rejection is surfaced on a stream a user reads AND in --json. Item 2: SATISFIED BY #3606 — validate_ms instruments validate_gpu_first_token, measured 15.1s on qwen2.5-coder-0.5b; not owed again here. Item 3: root cause named for the dense path — ANSWERED, it is FP8 PREFILL, established by a 2x2 over FP8_PREFILL x FP8_DECODE with both flags named in every cell (divergence tracks FP8_PREFILL exactly, independent of FP8_DECODE; 34.3s used_gpu=false vs 16.9s used_gpu=true); a single-knob bisect could not separate them because detect_fp8_decode falls through to fp8_prefill. Item 4: the fail-vs-fall-back decision RECORDED — it already was, in registry::after_generation (R-0b, gh#3002/#3042): forced accelerator that fell to CPU = refusal exit 14, default selection = corrective line. The defect was that it had NO PRODUCTION CALLER; nor do registry::announce or registry::parity_line, which is why a real run prints zero selected: and zero parity: lines. Item 5: falsifier both ways — 6 cases in run_tests_accel_reconcile.rs; planting the pre-fix behaviour turns the two defect tests RED while the healthy-GPU control stays green. Measured on lambda RTX 4090 sm_89, apr 0.68.2 (e6f77c98c). NOT IN SCOPE, deliberately: the rejection REASON (cosine, position) is not plumbed into --json — it is produced inside realizar F2 and no channel carries it to the CLI; that is a #3606-shaped follow-up, not a value to approximate. STOP: do not widen f2_should_retry_without_fp8 here — that band belongs to gh#3483, reopened with the contradicting measurement. BLAST RADIUS: exit code changes on a shipped surface; 14 files under scripts/ and .github/workflows/ pass --gpu, incl. perf_gate.sh, parity_host_receipt.sh, model_ladder.sh — CI going red on one is the measurement, and its fix belongs on that gate.' +- id: PMAT-3604 + github_issue: 3604 + item_type: task + title: 'f2_validate_qwen35 runs a full CPU reference forward on EVERY call (67% of a 14s TTFT) — run it once per (model sha256, apr version, device) and receipt it' + status: planned + priority: critical + assigned_to: null + created: 2026-09-20T22:25:52Z + updated: 2026-09-20T22:25:52Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — transcribed from issue #3604 done_when (minted by the cop so the AD-04 quorum for PR #3634 can run; pmat work status refused on a missing item). (1) Validation runs once per (model sha256, apr version, device); subsequent runs of the same triple read the receipt. (2) --revalidate forces a fresh validation and rewrites the receipt. (3) Falsifier: a planted receipt carrying the wrong model sha256 must cause re-validation, not a skip. Same for a receipt written by a different apr version, and one written for a different device. Three planted-receipt cases, each asserted to re-validate. (4) A missing or unreadable receipt validates — absence is never consent. (Unknown/skip is not an option here; the fallback is to do the work.) (5) validate_ms in apr run --json (#3598) reports the cached case distinguishably from the fresh case, so the saving is readable from the receipt rather than inferred from a stopwatch. (6) Measured before/after TTFT on the 144-word row, on the same box with GPU occupancy recorded at start. ADMISSION: 0.69 if it is green by the cut, 0.70 otherwise — readiness rule, not date pressure. EXPLICITLY OUT OF SCOPE (three questions the instrument lane raised and did not answer; none block this row and none should be folded in): whether the CPU reference or the GPU probe dominates the 9.5 s (timed as one unit); whether a cheaper equivalent guard exists; the flat ~1.18 s unattributed_ms residual — recorded rather than absorbed into a neighbour, and the next stage to name. Refs #3596, #3598, #3080.' +- id: PMAT-3605 + github_issue: 3605 + item_type: task + title: 'removed_by gets a shape: refusal-receipt-v1 with a closed escape set' + status: in_progress + priority: medium + assigned_to: null + created: 2026-09-21T00:00:00Z + updated: 2026-09-21T00:00:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3605 done_when; see paiml-mcp-agent-toolkit#1414). Done when: contracts/refusal-receipt-v1.yaml exists with a shape over removed_by, and the value space is a CLOSED SET rather than free text: pattern ^(v[0-9]+\\.[0-9]+|never|unscheduled)$. The point of the closed set is that legitimately-nothing stays SAYABLE and CHECKABLE — never and unscheduled are honest answers with a shape behind them, instead of a required field manufacturing a fake version. Fixtures cover each arm of the alternation plus a rejecting case. STOP: do not admit free text; do not make the field optional, which would make absence indistinguishable from a deliberate never.' +- id: PMAT-3618 + github_issue: 3618 + item_type: task + title: 'Release scripts still carry 0.68.2''s V/T/MS/EPIC and AP=/mnt/... — and the hardcoded-path ratchet does not match /mnt, so nothing catches it' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:36:49Z + updated: 2026-09-21T05:36:49Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3618 "done_when"): 1. V/T/MS/EPIC are derived or passed, never literals. 2. AP is derived from the repository, not from a path under /mnt. The scripts are in-tree now; an out-of-tree bundle path is a leftover of where they used to live. 3. Either the /mnt blind spot closes (#3592) or this row ships its own assertion that no release script references a path outside the repo. Prefer the first — the blind spot affects every script, not these. 4. Falsifier: reintroduce a literal /mnt/... in a release script and require a gate to go RED. If nothing can be made to go red, item 3 is unsatisfied regardless of what was edited. SCOPE NOTE (author): this row ships its own assertion (item 3, second branch); #3592 stays open for the analyser itself.' - id: PMAT-3627 github_issue: 3627 item_type: task @@ -19007,3 +19723,166 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from issue #3629 done_when; minted on the PR branch so the AD-04 quorum can run -- pmat work status refuses a missing item). Done when: scripts/fleet_utilization.sh --selftest has no pipe into grep -q; a mutation that restores the pipe AND forces EPIPE (out padded to >64 KiB so the write cannot complete before grep exits) turns the self-test RED deterministically — the positive control for the race, in the PR body the 55-site count is in the PR body from the command that produced it, and a guard refuses a new site (check_guards_are_wired.sh names it; ratchet baseline committed) main''s guard-tree green on the merge commit without rerun ' +- id: PMAT-3637 + github_issue: 3604 + item_type: task + title: 'Register PMAT-3604 in the roadmap so the AD-04 quorum for PR #3634 can run (registration only; the implementation is PR #3634)' + status: planned + priority: high + assigned_to: null + created: 2026-09-20T22:39:43Z + updated: 2026-09-20T22:39:43Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:triage + notes: 'ACCEPTANCE (this row is PR #3637 itself, a roadmap-only registration): (1) docs/roadmaps/entries/PMAT-3604.yaml exists with id PMAT-3604, github_issue 3604, kind:code, and its notes transcribe issue #3604 done_when items 1-6 faithfully — every criterion present, none added, the admission rule and the out-of-scope list as the issue states them; (2) docs/roadmaps/roadmap.yaml equals aggregate(entries/) and the diff vs origin/main is additive only (deleted=0, reserialised=0); (3) the PR body carries keep-open for #3604 and the diff touches nothing outside docs/roadmaps/ except this row''s own quorum receipt(s) at docs/audits/quorum-PMAT-3637*.json. NOT in scope: any code, test or measurement of PMAT-3604 — those are PR #3634 and are judged there. A lane that finds a done_when item mis-transcribed, missing or invented FAILs this row.' +- id: PMAT-3644 + github_issue: 3644 + item_type: task + title: 'check_guards_are_wired.sh reads a step NAME as a bare guard_tree.sh dispatch: 4 guards dark, incl. the #3305 publish-strip guard (which also dies on intel''s python 3.10)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T01:39:50Z + updated: 2026-09-21T01:39:50Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3644 "What lands"; minted on the PR branch so the AD-04 quorum can run). 1. check_guards_are_wired.sh drops name: lines before the invocation match, in dispatcher_wired() and the per-guard scan; rows 8-10 (the exact ci.yml step-name fixture must NOT count as a dispatch; a quoted run: line still does; a step named after a guard wires nothing); the mutation (remove the drop) turns rows 8 and 10 RED. 2. check_pathonly_devdeps_unused_in_src.sh: tomllib -> tomli -> ENV rc=2 naming interpreter and runner; the scanner ends with a SCAN-DONE trailer the shell REQUIRES; five death rows incl. the whole guard under the intel shape; comments reworded so CARGO_RE no longer classifies it out of --no-cargo, so guard-tree runs it (bare run on the tree: rc 0, 3 s). 3. unwired_guards_baseline.txt records the other three dark guards as APERTURE REVEALS (set-aperture gains a NAME-entry admission: a file the comparand carries, in a diff that changes the owning guard; six case rows, each admission beside its refusal) with the reason each may only leave; #3646 owns the pr_review pair. No workflow file is edited.' +- id: PMAT-3651 + github_issue: 3651 + item_type: task + title: 'guard_tree.sh drops a passing guard''s output — an UNMEASURED exit-0 reads as PASS in every CI log, so #3567''s verdict can never be observed' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T05:28:51Z + updated: 2026-09-21T05:28:51Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3651 "What lands"): 1. guard_tree.sh: under a PASS row, surface every captured line whose first token is UNMEASURED or SUMMARY (indented " ~ ", capped at 5 lines + a "(+N more)" count so a chatty guard cannot flood the log). Lines where the word is not the first token are not surfaced. A guard opts a pass-summary in by printing SUMMARY …; an UNMEASURED line is surfaced whether or not the guard opted in. 2. check_fleet_pv_shapes_gate.sh: its pass row is printed as SUMMARY PASS verdict=… so the measured row reaches the log. 3. scripts/tests/guard_tree_test.sh rows: UNMEASURED surfaced; SUMMARY surfaced; the word mid-line not surfaced; a silent pass prints only its PASS row; the cap; and a mutant with the surfacing removed goes RED. NOT in scope: closing #3567 (that needs one guard-tree run on an intel runner after this lands), and the fleet pv''s own verdict.' +- id: PMAT-3656 + github_issue: 3656 + item_type: task + title: 'apr convert/import invent IQ-quant weights: get_tensor_f32 approximated IQ2..IQ4 from raw bytes and exited 0' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T05:47:55Z + updated: 2026-09-21T05:47:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3656 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). Done when: (1) get_tensor_f32 on any IQ tensor returns Err naming the tensor and its type; no path in aprender-core returns approximated IQ values; dequantize_iq_approximate is deleted. (2) apr convert … --quantize q4k and apr import on Qwen3.5-0.8B-IQ4_XS.gguf exit non-zero and write no output file. (3) A unit test fails if an approximation arm is restored (mutation: restore the 16..=23 arm → RED). Not in scope: real IQ dequantization in aprender-core (aprender-serve has dequantize_iq*, core cannot depend on serve; the route is PP-QUANT-001 #3421). Why it rides with PMAT-3601: once IQ tensors are sized, raw apr import reaches map_gguf_dtype, whose Q8_0/Q5_K refusal says "cannot represent exactly" — a GH-375 fallback trigger — so without this fix an IQ file could import invented weights where it used to be refused.' +- id: PMAT-3658 + github_issue: 3658 + item_type: task + title: 'Every merge-group run is tier=full: ci_test_tier.sh requires HEAD^2 but the queue is SQUASH (one parent) — a text-only PR runs ~2 runner-hours of unit tests; HEAD^1..HEAD is already the PR diff' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T06:03:14Z + updated: 2026-09-21T06:03:14Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — issue #3658 done_when as AMENDED 2026-09-21 06:10Z (the amendment supersedes the original list; operator instruction quoted there: "you will fix ASAP this is P0, and NO roadmap shouldn''t even have 10 minutes testing"). (1) A squash-shaped merge-group fixture (one parent, main moved, a leaf crate touched) gives tier=quick with that crate, not full. The same fixture with a roadmap entry only gives tier=none. Both are case-table rows. (2) Mutation: restoring the HEAD^2 gate sends the squash group back to full (a RED row). (3) A failing git diff HEAD^1..HEAD still gives tier=full (the fail-closed row is kept). (4) On a pull_request: roadmap + audit paths only -> tier=none; the same with a deleted docs path -> quick; roadmap plus any non-docs path -> quick; docs/specifications/ only -> quick. (5) A push of a squash landing (one parent, no reflog) -> quick over HEAD^1..HEAD, not full. (6) [AMENDMENT 2, 2026-09-21 06:55Z, supersedes amendment 1 item 6] ci.yml changes in exactly two places, and nothing else in the workflow changes: (a) one step, if: tier == none, that logs the reason through env:; (b) the pull_request touched-list line (git diff ... > "$RUNNER_TEMP/touched.txt" in the tier step) gains --no-renames, so both sides of every rename are touched paths. diff_into in ci_test_tier.sh gains the same flag. Rows: default --name-only on a rename fixture lists only the new path (the trap is real); the rename into docs/audits/ gives quick with the crate it left; a mutant without --no-renames gives none (RED); and a file-shaped row asserts the ci.yml line carries the flag. The other touched-list builder with the same shape (ci.yml:2783, the ci_gpu_touched feed) is pre-existing, has a different consumer, is out of scope, and is #3667. (7) After landing: the next 5 merge-group runs'' shard-1 logs show a tier= other than full for the docs-only and small-crate groups. This is observed after the merge, and the issue stays open until it is. SCOPE ADDITIONS in the amendment: A. docs-only diffs (every path under docs/roadmaps/ or docs/audits/ and present at HEAD; never a delete) and an empty diff get tier=none; B. a push landing a squash uses HEAD^1..HEAD. NOT IN SCOPE: guard-cargo and the reusable ci/* jobs on docs-only changes (the next row). Refs #3650, #3643.' +- id: PMAT-3662 + github_issue: 3662 + item_type: task + title: 'Hand-typed ggml id→name tables outside aprender-core: sliding_window_entropy.rs, validate_manifest.rs, aprender-serve model_source.rs name 11 of 35 live types; route through trueno_quant::GgmlType' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T06:40:00Z + updated: 2026-09-21T06:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3662 done_when 1, verbatim: "Every non-example id→name mapping in `apr-cli` and `aprender-serve` resolves through `trueno_quant::GgmlType::from_id(id).map_or(, GgmlType::as_str)`, or through one shared helper that does. No `12 => "Q4_K"`-shaped match arm remains outside `aprender-quant` in those two crates'' `src/`."' + - 'Issue #3662 done_when 2, verbatim (as amended by the filer 2026-09-21 after measuring): "A test per site feeds ids 10 (Q2_K), 16 (IQ2_XXS), 23 (IQ4_XS) and 30 (BF16) and gets the upstream names. It is RED on today''s tables where they are incomplete (sliding_window_entropy.rs, model_source.rs). validate_manifest.rs was complete when measured (amended 2026-09-21 by the filer), so its row is a parity row and passes before and after."' + - 'Issue #3662 done_when 3, verbatim: "Unknown ids (e.g. 4, 5, 999) still print the site''s existing unknown token, so output stays stable for ids ggml removed."' + - 'Issue #3662 done_when 4, verbatim: "`apr-cli` depends on the quant crate via `{ workspace = true }` (no crates.io copy)."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-7a (sliding_window_entropy.rs:295); the other two sites found by aprender-62 at origin/main c0ab699b0 with a scan of crates/apr-cli/src and crates/aprender-serve/src for ` => "Q4_K"`-shaped arms, which returns exactly three: apr-cli commands/sliding_window_entropy.rs `ggml_dtype_name` (unknown token "Unknown"; knows 11 ids), apr-cli commands/validate_manifest.rs `ggml_type_name` (unknown token "UNKNOWN"; knows ids 0-3,6-30, so it is complete today but hand-typed), aprender-serve api/model_source.rs `gguf_qtype_name` (returns None; knows 15 ids, so every IQ*, I8-I64 and F64 is None). "RED on today''s tables" is per site. validate_manifest.rs already names 10/16/23/30 correctly, so its row (ids 10/16/23/30 plus unknown ids 4/5/999, the same ids as the other two sites) is a parity row and is not expected to be RED. The two tables that are wrong today (sliding_window_entropy, model_source) carry the RED rows. validate_manifest.rs''s reverse name→id parser (`"iq4_xs" => Some(23)`) has case-insensitive aliases ("bfloat16") and is not an id→name mapping. It is out of scope and stays. Examples under crates/aprender-serve/examples/ are out of scope per the issue. + + Facts a reviewer needs, each checked at the PR head (round 1 of the quorum misread all three). (1) WHERE sliding_window_entropy.rs IS TESTED: it is not a module. crates/apr-cli/src/commands/hex.rs:442 is `include!("sliding_window_entropy.rs")`, and hex.rs:444 includes hex_06.rs, whose :11 is `include!("hex_options_detect.rs")`. So `ggml_dtype_name` and its tests share one module, `commands::hex`. The pre-existing tests for this function (test_ggml_dtype_name_known/_unknown) already live in hex_options_detect.rs, and the new test_ggml_dtype_name_names_every_family sits beside them. It ran as commands::hex::tests::test_ggml_dtype_name_names_every_family: RED on the old table (left "Unknown", right "Q2_K"), GREEN after. (2) THE CRATE NAME: the workspace dependency is `trueno-quant = { path = "crates/aprender-quant", ..., package = "aprender-quant" }` (root Cargo.toml:262), and Cargo.lock records PACKAGE names. So `aprender-quant` in apr-cli''s Cargo.lock dependency list is what cargo wrote for `trueno-quant = { workspace = true }`. It was generated by cargo, not hand-edited. (3) GgmlType IS EXPORTED: crates/aprender-quant/src/lib.rs has an unconditional `pub mod ggml_type;` and `pub use ggml_type::{GgmlFamily, GgmlType, ...}`. The type was added by #3581 AFTER the v0.68.2 tag (`git cat-file -e v0.68.2:crates/aprender-quant/src/ggml_type.rs` fails), so a tree at or before v0.68.2, or the crates.io aprender-quant 0.68.2, does not have it and fails with E0433. At the PR head, `cargo check -p aprender-serve` and `cargo check -p apr-cli` both exit 0. Verified at the PR head: cargo test -p aprender-serve --lib model_source 19 passed; cargo test -p apr-cli --lib -- commands::hex commands::validate_manifest 174 passed; clippy -D warnings clean on both crates.' +- id: PMAT-3670 + github_issue: 3670 + item_type: task + title: 'check_fleet_pv_shapes_gate.sh FAILs guard-tree on fleet state (intel pin rewritten to 0.65.2 by another repo''s CI) — fleet state must be UNMEASURED, never FAIL' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T07:33:19Z + updated: 2026-09-21T07:33:19Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE — issue #3670 done_when, verbatim in substance. (1) Each fleet-state branch prints UNMEASURED runner=... reason= ..., exits 0, and never prints PASS or SUMMARY PASS. (2) A mutant restoring FAIL on pin-mismatch exits 1, and a mutant restoring the ENV exit on pv-env exits 2; each mutant row refuses to pass if its edit did not apply. (3) A planted violation from a matching pinned binary is still RED, and a control that does not fire is still RED. (4) Real runs: pin forced to 0.65.2 under a 0.68.2 binary gives UNMEASURED reason=pin-mismatch (rc 0); the fleet pv on a tree with migrated parity records gives PASS verdict=Pass. RULING: PASS/FAIL only from a verdict produced by a binary that matches its pin; fleet state is UNMEASURED with a named reason and exit 0. Scope: scripts/check_fleet_pv_shapes_gate.sh only, one commit on top of #3653 (053992448).' +- id: PMAT-3681 + github_issue: 3681 + item_type: task + title: '#3669 batch: two functions over the complexity ratchet (run_shapes_gate NEW 13/28, dispatch_runtime_commands GROWN 41->43) — extract helpers, no behaviour change' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T09:12:54Z + updated: 2026-09-21T09:12:54Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3681): 1. run_shapes_gate is under the threshold (cyclomatic < 30, cognitive <= 25) and dispatch_runtime_commands is back to main''s 15/41 — bash scripts/check_complexity_ratchet.sh reports none new, none grown. 2. No behaviour change: each extraction moves code verbatim into a helper (same order, same outcomes); cargo test -p aprender-contracts --lib passes the same count as before (1684); cargo test -p apr-cli --lib dispatch/registry pass; clippy -D warnings clean on both crates. 3. Nothing else in the diff: only shapes_gate.rs and dispatch.rs, plus this row''s roadmap fragment. CONTEXT: the base of this diff is the batch head 01abbca6d (release/0.69-batch); the functions'' prior shapes come from #3600 (run_shapes_gate) and #3604 + #3638/#3602 (dispatch_runtime_commands), which are already in the batch with their own receipts and are NOT judged here.' +- id: PMAT-3682 + github_issue: 3682 + item_type: task + title: 'parity_receipt_denominator.sh reads git''s refusal as 0 receipts on container runners — a git failure must be ENV exit 2, never a count' + status: in_progress + priority: critical + assigned_to: aprender-62 + created: 2026-09-21T09:19:00Z + updated: 2026-09-21T09:19:00Z + spec: null + acceptance_criteria: + - 'Issue #3682 done_when 1, verbatim: "The listing is `git -c safe.directory="$root" -C "$root" ls-files …`, and git''s stderr is no longer discarded."' + - 'done_when 2, verbatim: "A git failure makes the script exit **2 (ENV)** with a message naming git. It is never a count of 0. `verify` propagates that status instead of collapsing it to 1."' + - 'done_when 3, verbatim: "A self-test row puts a `git` stub on PATH that exits 128 with the dubious-ownership message. It requires exit 2 and a message naming git, and it requires that the output NOT contain \"holds 0\"."' + - 'done_when 4, verbatim: "Mutation: with the pre-fix `count_and_check` restored, that row goes RED."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'The judged diff is this ONE commit, scripts/parity_receipt_denominator.sh plus this fragment, with --base at its parent. The three commits under it on the 0.69 batch branch (the dogfood ledger re-audit, the ONT ratchet restamp, the ont4b 6->9 raise) are data/ratchet restamps that carry their proofs in their commit bodies and are outside this ticket. Found in batch #3669, workspace-test shard 3/3 on gx10-pool1 (job 106269529629): "EXPECTED_RECEIPTS says 7; the tree holds 0". On the docker runners git refuses the bind-mounted tree ("detected dubious ownership", exit 128), and the old `(cd "$root" && git ls-files … 2>/dev/null | sort -u)` inside a process substitution discarded both the message and the exit status. Measured at this commit: `bash scripts/parity_receipt_denominator.sh --self-test` 5/5 ok (the new row: "git refusing the tree is ENV exit 2 naming git, never a count of 0"); `bash scripts/parity_receipt_denominator.sh` "PASS 7 receipt(s) … says 7"; MUTANT with the pre-fix count_and_check from 55b123f2f: self-test rc=1, "FAIL git refused and the result was exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0."; `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` 9 passed. Sibling sweep, on lines the batch ADDED in scripts/ and crates/*/tests/: ci_test_tier.sh:180 and :252 fail closed (not docs-only / tier=full); lib_baseline_ratchet.sh:300/302 fail closed but misattribute the cause ("this branch WROTE it"). No other zero-count site. Outside test paths: release/lib_release_params.sh:39 silently falls back to the tree root. Those are filed, not fixed here.' diff --git a/docs/specifications/06x-release-schedule.md b/docs/specifications/06x-release-schedule.md index d4354e7659..0671c61840 100644 --- a/docs/specifications/06x-release-schedule.md +++ b/docs/specifications/06x-release-schedule.md @@ -279,7 +279,7 @@ The milestone already holds 15 issues; with the 14 rows above the train is over ## §4. Release-day protocol (every train; the FULL tier's only mandatory home) -Numbered so an autopilot (`/mnt/nvme-raid0/agent-wt/rel-066-autopilot/` is the 0.66 precedent) can run it fail-closed, with a `STOP ` line naming the refusing gate. +Numbered so an autopilot can run it fail-closed, with a `STOP ` line naming the refusing gate. Since #3618 that autopilot is in-tree: `scripts/release/autopilot.sh `, with `prepare_bump.sh [--ship]`, `t2_preflight.sh ` and `publish_strict.sh [--plan]` beside it. The version is the only input; the milestone, the release epic, the previous tag and the state directory (`
/target/release-train/v`) are derived by `scripts/release/lib_release_params.sh`, and `scripts/check_release_scripts_derive_identity.sh` refuses a literal identity or an out-of-repo path in that directory. (The 0.66–0.68.2 trains ran from out-of-tree bundles.) 1. **Freeze the platform.** `gh pr list --state open --label `: rows still open move to the next epic with `slipped_from:`. No new pushes while the queue drains (`[[feedback_push_cadence_starves_the_merge_queue]]`). Then read the milestone, before the bump PR opens: `bash scripts/check_milestone_cut.sh X.Y.0` must exit 0, meaning no open issue **or** PR except this train's own release epic (§5), which is open until step 8 by design and is named as `ADMITTED`. An item that does not ride this train is **carried by moving it**: `gh issue edit N --milestone ` (or `gh pr edit`) plus a `slipped_from: X.Y.0` comment. It is never carried by leaving it open here, because step 8 closes the milestone, and the train driver refuses to close one whose `open_issues` is not 0 (PMAT-3445, #3445). 2. **Bump.** `bash scripts/bump-version.sh X.Y.0`; `CHANGELOG.md` section from the epic's rows; the bump PR carries the receipt(s). diff --git a/docs/specifications/crux-competitive-research-ux-workflows.md b/docs/specifications/crux-competitive-research-ux-workflows.md index b556c0d96f..5ec6ca6045 100644 --- a/docs/specifications/crux-competitive-research-ux-workflows.md +++ b/docs/specifications/crux-competitive-research-ux-workflows.md @@ -2,8 +2,8 @@ **Subspec ID**: `CRUX-001` **Status**: DRAFT -**Version**: 2.2 (2026-04-21 — Category L [HF kernels-community, 15 stories] and Category M [APR-QA Playbook canonicalization, 10 stories] added; §13 chain-of-thought derivation appended; §3 matrix and §6 coverage recomputed; story total 250 → 275) -**Date**: 2026-04-21 +**Version**: 2.3 (2026-09-16 — Category O [AutoGluon AutoML parity, 24 stories] added, epic aprender#3370; Category N [linfa + burn, 17 stories, aprender#3146, 2026-09-12] exists in the master contract and is recorded here in §3 for the first time; story total in the master registry 267 → 291) +**Date**: 2026-09-16 **Author**: PAIML Engineering **Parent**: [aprender-spec.md](aprender-spec.md), [aprender-monorepo-consolidation.md](aprender-monorepo-consolidation.md) **Master contract**: [`contracts/crux-competitive-research-ux-v1.yaml`](../../contracts/crux-competitive-research-ux-v1.yaml) @@ -93,8 +93,10 @@ larger workflow surface area (HF Transformers covers training + data + hub). | 7 | **Ecosystem interop** | — | 30 | SDKs, MCP, observability, deployment | | 8 | **HF kernels-community** | `get_kernel("kernels-community/")` | 15 | optimized GPU kernels as drop-in `.so` packages (v2.2) | | 9 | **APR-QA Playbook** | `apr qa --gate=` / `apr-model-qa-playbook` | 10 | Popperian falsification framework for model qualification (v2.2) | +| 10 | **linfa + Burn** | `linfa::traits::Fit` / `burn::module::Module` | 17 | Rust-native ML frameworks; substrates (SVD, spatial index, rank-typed tensor) + breadth (Category N, 2026-09-12, aprender#3146) | +| 11 | **AutoGluon** | `TabularPredictor(label).fit()` / `TimeSeriesPredictor.fit()` | 24 | AutoML: one-call tabular fit, presets, leaderboard, bagging/stacking/weighted ensemble, panel forecasting with quantiles (Category O, v2.3, aprender#3370) | -Total = 275 stories. See §5 for the full registry. +Total = 275 stories in this document's §5 as of v2.2; the master registry carries 291 (Categories L and M are contract-only, see §6 note). See §5 for the full registry. (Counts derived from `yq '[.stories[] | .competitor] | ...'` on master contract; drift between this table and the YAML is falsified by FALSIFY-CRUX-010.) @@ -501,7 +503,38 @@ always `contracts/crux-{ID}-v1.yaml` unless noted. | CRUX-M-09 | Property-based falsifier ≥ 1000 fuzz cases per gate | `apr-qa fuzz --cases 1000` | ❌ | 4 | | CRUX-M-10 | Upstream-fix enforcement (reject workarounds; route to aprender/trueno/realizar) | playbook "no-workarounds" rule | 🔨 | 4 | -**Total: 275 stories** across 13 categories; 5 ID gaps (`C-14`, `F-10`, `H-04`, `I-05`, `K-06`) intentional and documented. +### Category O — AutoML Parity, AutoGluon (24 stories) + +> Added v2.3 (2026-09-16). Competitor source: [autogluon/autogluon](https://github.com/autogluon/autogluon) 1.6.3, surveyed from `../autogluon` @ 77946149; evidence in `evidence/crux/autogluon/`. Canonical verb: `TabularPredictor(label="class").fit("train.csv", presets="best")` — the README's only code block. Aprender target surface: `apr automl fit|predict|leaderboard` over a predictor-level AutoML in `aprender-core::automl` (today a single-estimator tuner) and `apr forecast` over a panel `TimeSeriesPredictor` (today one univariate `ARIMA`). Epic aprender#3370; one GitHub issue per row (#3371–#3394). `MultiModalPredictor`, `autogluon.cloud`, MLZero and Ray-parallel fits are CUT on the epic. Not a BEAT pillar. + +| ID | Story | Competitor verb | S | D | +|----|-------|----------------|---|---| +| CRUX-O-01 | One-call tabular AutoML: fit(label) -> predict on a CSV | `TabularPredictor(label="class").fit("train.csv"); predictor.predict("test.csv")` | ❌ | 5 | +| CRUX-O-02 | Problem-type inference: binary / multiclass / regression / quantile from the label column | `predictor.problem_type (inferred in fit unless problem_type= given)` | ❌ | 5 | +| CRUX-O-03 | Quality presets (medium / good / high / best / extreme) that name a model portfolio and a time budget | `fit(..., presets="best_quality")` | ❌ | 5 | +| CRUX-O-04 | Leaderboard: per-model validation/test score, fit time, predict time and stack level | `predictor.leaderboard(test_data, extra_info=True)` | ❌ | 5 | +| CRUX-O-05 | Automatic feature-type inference and the AutoML feature pipeline (numeric, categorical, datetime, text n-gram, drop-unique, drop-duplicate) | `AutoMLPipelineFeatureGenerator` | 🔨 | 5 | +| CRUX-O-06 | K-fold bagging with out-of-fold predictions (num_bag_folds, predict_oof) | `fit(..., num_bag_folds=8); predictor.predict_proba_oof()` | ❌ | 4 | +| CRUX-O-07 | Multi-layer stack ensembling with a leakage guard (num_stack_levels, auto_stack, dynamic_stacking) | `fit(..., num_stack_levels=1, dynamic_stacking="auto")` | ❌ | 4 | +| CRUX-O-08 | Greedy weighted-ensemble selection over fitted models (Caruana ensemble selection) | `fit_weighted_ensemble=True (default)` | ❌ | 5 | +| CRUX-O-09 | Time-budgeted portfolio fit: time_limit split across models, each model early-stopped on its share | `fit(..., time_limit=3600)` | 🔨 | 5 | +| CRUX-O-10 | Predictor-level permutation feature importance with p-values and confidence intervals | `predictor.feature_importance(test_data, num_shuffle_sets=10)` | 🔨 | 4 | +| CRUX-O-11 | Decision-threshold calibration for binary metrics (calibrate_decision_threshold) | `fit(..., calibrate_decision_threshold="auto"); predictor.calibrate_decision_threshold(metric="f1")` | 🔨 | 4 | +| CRUX-O-12 | refit_full: retrain the selected models on train+validation after model selection | `fit(..., refit_full=True, set_best_to_refit_full=True); predictor.refit_full()` | ❌ | 3 | +| CRUX-O-13 | Model distillation: compress the ensemble into one fast student | `predictor.distill(time_limit=..., augment_method="spunge")` | ❌ | 3 | +| CRUX-O-14 | Deployment artifact: clone_for_deployment / keep_only_best / save_space / persist into one loadable file | `predictor.clone_for_deployment(path); predictor.persist()` | ❌ | 4 | +| CRUX-O-15 | Inference-latency constraint during model selection (infer_limit, infer_limit_batch_size) | `fit(..., infer_limit=0.001, infer_limit_batch_size=10000)` | ❌ | 3 | +| CRUX-O-16 | Fit diagnostics: fit_summary, model_failures and learning curves | `predictor.fit_summary(); predictor.model_failures(); fit(..., learning_curves=True)` | ❌ | 3 | +| CRUX-O-17 | Tabular foundation model: in-context prediction with a pretrained transformer (TabPFN / TabICL / Mitra class) | `hyperparameters={"TABPFNV2": {}, "TABICL": {}, "MITRA": {}}` | ❌ | 4 | +| CRUX-O-18 | Memory-aware fit: per-model memory estimate and a memory_limit that skips models that would not fit | `fit(..., memory_limit="auto")` | ❌ | 3 | +| CRUX-O-19 | Multi-series forecasting predictor: (item_id, timestamp) panel data, prediction_length, freq | `TimeSeriesPredictor(prediction_length=48, freq="h").fit(TimeSeriesDataFrame)` | ❌ | 5 | +| CRUX-O-20 | Probabilistic forecasts: quantile_levels and the forecasting metric family (WQL, MQL, MASE, SMAPE, RMSSE, WAPE) | `TimeSeriesPredictor(eval_metric="WQL", quantile_levels=[0.1,0.5,0.9])` | ❌ | 5 | +| CRUX-O-21 | Rolling-window backtesting: num_val_windows, refit_every_n_windows, backtest_predictions | `fit(..., num_val_windows="auto", refit_every_n_windows="auto"); predictor.backtest_predictions()` | ❌ | 4 | +| CRUX-O-22 | Known covariates, past covariates and static features in forecasting | `TimeSeriesPredictor(known_covariates_names=["holiday"]); train_data.static_features = df` | ❌ | 3 | +| CRUX-O-23 | Local statistical baselines: SeasonalNaive, ETS, Theta, AutoARIMA, Croston | `hyperparameters={"SeasonalNaive": {}, "AutoETS": {}, "Theta": {}, "AutoARIMA": {}, "Croston": {}}` | 🔨 | 4 | +| CRUX-O-24 | Zero-shot pretrained forecaster (Chronos-2 / Toto-2 class) loaded by apr pull, with optional fine-tuning | `hyperparameters={"Chronos2": {"fine_tune": True}}` | ❌ | 4 | + +**Total: 275 stories in §5 (v2.2) + 24 in Category O = 299 documented rows; the master registry holds 291** (Categories L and M have contracts but no registry rows — pre-existing drift recorded on aprender#3146) across 15 categories; 5 ID gaps (`C-14`, `F-10`, `H-04`, `I-05`, `K-06`) intentional and documented. --- @@ -517,6 +550,8 @@ Counts verified from §5 table (via `awk` emoji extraction). Δ columns show v2. | 🤔 unclear | 0 | 0 | 0.0 % | — | | **total** | **275** | +25 | 100 % | | +> **v2.3 note (2026-09-16):** the master contract's `coverage_intake` is now asserted from `stories[]` by the aprender-contracts test-suite and reads supported 43 / partial 77 / missing 171 / total 291 after Category O (+5 partial, +19 missing). This table is the v2.2 intake and is left as history; the YAML is the source of truth. + Demand-weighted view — **high-demand (D≥4)** stories still ❌ missing are the fast path to adoption parity and become the first `pmat work` items (see §12). Exact D-tier counts are regenerated by the falsification harness diff --git a/evidence/crux/autogluon/README.md b/evidence/crux/autogluon/README.md new file mode 100644 index 0000000000..81e97763b9 --- /dev/null +++ b/evidence/crux/autogluon/README.md @@ -0,0 +1,3 @@ +# evidence/crux/autogluon + +CRUX Category O evidence: AutoGluon 1.6.3 (`../autogluon` @ 77946149), surveyed 2026-09-16. See `api-surface.md` (the survey), `readme-verbs.txt` (fold-ranked verbs), `hello.sh` (canonical flow). Epic aprender#3370. diff --git a/evidence/crux/autogluon/api-surface.md b/evidence/crux/autogluon/api-surface.md new file mode 100644 index 0000000000..2e74a185c8 --- /dev/null +++ b/evidence/crux/autogluon/api-surface.md @@ -0,0 +1,46 @@ +# AutoGluon 1.6.3 — user-facing surface (evidence for CRUX Category O) + +Surveyed 2026-09-16 from `../autogluon` @ 77946149 (`VERSION` = 1.6.3). Paths are repo-relative to the autogluon checkout. + +## TabularPredictor (`tabular/src/autogluon/tabular/predictor/predictor.py`) + +Public methods (65): fit, fit_extra, fit_pseudolabel, predict, predict_proba, predict_from_proba, evaluate, evaluate_predictions, leaderboard, learning_curves, model_failures, predict_multi, predict_proba_multi, fit_summary, transform_features, transform_labels, feature_importance, compile, persist, unpersist, refit_full, model_best, set_model_best, model_refit_map, info, model_info, model_hyperparameters, fit_weighted_ensemble, calibrate_decision_threshold, set_decision_threshold, predict_oof, predict_proba_oof, save_space, delete_models, disk_usage, model_names, distill, plot_ensemble_model, save, load, load_log, clone, clone_for_deployment, simulation_artifact, confusion_matrix, plus properties (problem_type, eval_metric, decision_threshold, feature_metadata, class_labels, positive_class, quantile_levels). + +`fit` kwargs that carry a story: presets, time_limit, hyperparameters, num_bag_folds, num_bag_sets, num_stack_levels, auto_stack, dynamic_stacking, fit_weighted_ensemble, refit_full, set_best_to_refit_full, save_bag_folds, keep_only_best, holdout_frac, use_bag_holdout, infer_limit, infer_limit_batch_size, calibrate_decision_threshold, learning_curves, memory_limit, num_cpus, num_gpus, fit_strategy, feature_generator, excluded_model_types, included_model_types, raise_on_no_models_fitted, callbacks, core_kwargs/aux_kwargs (1.6). + +Presets (`tabular/src/autogluon/tabular/configs/presets_configs.py`): extreme_quality (zeroshot portfolio, 8 bag folds, foundation models), best_quality (auto_stack + dynamic_stacking), high_quality (+ refit_full, no bag folds saved), good_quality (light portfolio), medium_quality (no bagging), optimize_for_deployment (keep_only_best + save_space), ignore_text, ignore_text_ngrams, interpretable, noncommercial, tabarena. Portfolios: `configs/zeroshot/zeroshot_portfolio_{2023,2025,cpu_2025_12_18,gpu_2025_12_18,commercial_2026_08_05,noncommercial_2026_08_05}.py`. + +Model families (`tabular/src/autogluon/tabular/models/`): catboost, ebm, fastainn, imodels, knn, lgb, lr, mitra, nori, realmlp, rf, tabdpt, tabicl, tabm, tabpfnmix, tabpfnv2, tabprep, tabular_nn, xgboost, xt (+ automm, image_prediction, text_prediction wrappers). The 2026 commercial portfolio names CAT, GBM, XGB, MITRA, TABICL, TABM. + +Feature pipeline (`features/src/autogluon/features/generators/`): auto_ml_pipeline (enable_numeric/categorical/datetime/text_special/text_ngram/raw_text/vision features), astype, binned, category, cat_int, datetime, drop_duplicates, drop_unique, fillna, frequency, groupby, isnan, label_encoder, one_hot_encoder, oof_target_encoder, text_ngram, text_special, memory_minimize, skrub, rsfc, selection. + +## TimeSeriesPredictor (`timeseries/src/autogluon/timeseries/predictor.py`) + +Constructor: target, known_covariates_names, prediction_length, freq, eval_metric, eval_metric_seasonal_period, horizon_weight, quantile_levels, cache_predictions (deprecated 1.6), log_to_file. +fit: train_data (TimeSeriesDataFrame with item_id/timestamp index + static_features), tuning_data, time_limit, presets, hyperparameters, hyperparameter_tune_kwargs, excluded_model_types, ensemble_hyperparameters, num_val_windows ("auto" since 1.5), val_step_size, refit_every_n_windows ("auto"), refit_full, enable_ensemble, skip_model_selection, random_seed. +Methods: predict, backtest_predictions, backtest_targets (1.5), evaluate, feature_importance, leaderboard, fit_summary, refit_full, persist/unpersist, export_model (1.6: standalone checkpoint), update (1.6 experimental: ensemble re-selection), make_future_data_frame, plot. +Metrics (`timeseries/.../metrics/point.py`, `quantile.py`): MQL WQL SQL RMSE MSE MAE MAEB BIAS WAPE WAPEB SMAPE MAPE MASE RMSSE RMSLE WCD (MAEB/WAPEB/BIAS/MQL new in 1.6). +Models: local (Naive, SeasonalNaive, Average, SeasonalAverage, NPTS, Zero; statsforecast AutoARIMA/ARIMA/AutoETS/ETS/AutoCES/Theta/DynamicOptimizedTheta/Croston/ADIDA/IMAPA), gluonts (DeepAR, SimpleFeedForward, TFT, DLinear, PatchTST, WaveNet, TiDE), pretrained (Chronos, Chronos2 with LoRA/full fine-tune, Toto, Toto2), tabular (per-step, recursive/direct via mlforecast), ensembles (greedy selection, per-item greedy, weighted, array-based, multi-layer since 1.5). + +## MultiModalPredictor (`multimodal/src/autogluon/multimodal/predictor.py`) — CUT on epic #3370 + +Problem types: classification, regression, few_shot_classification, object_detection, ner / named_entity_recognition, image/text/image_text similarity (matching), semantic_segmentation, zero_shot_image_classification, document classification. Methods: fit, predict, predict_proba, evaluate, extract_embedding, export_onnx, optimize_for_inference, dump_model, list_supported_models. + +## Release headlines used for demand scoring + +- 1.4 (2025): extreme preset; TabPFNv2, TabICL, TabM, RealMLP; Mitra; MLZero (AutoGluon Assistant). +- 1.5: Chronos-2 with zero-shot + fine-tuning; item-level and multi-layer forecast ensembles; `num_val_windows="auto"`, `backtest_predictions`; RealTabPFN-2/2.5, TabDPT, TabPrep-LightGBM, EBM; new CPU/GPU portfolios; TabArena SOTA. +- 1.6: Nori, TabPFN-3, TabDPT-Turbo, TabPFN-2.6, TabICLv2; Toto-2; MAEB/WAPEB/BIAS/MQL metrics; `TimeSeriesPredictor.export_model`; `update()`; calibrated CPU/GPU memory estimates; GPU-aware parallel bagging; feature-importance memory cut 25x; params immutable after construction (deprecation). +- Docs index (`docs/tutorials/{tabular,timeseries,multimodal,cloud_fit_deploy}/index.md`): tabular quick start → essentials → in-depth → feature engineering → foundational models → multimodal → FAQ; timeseries quick start → in-depth → Chronos → ensembles → metrics → model zoo → FAQ. + +## What aprender has today (eb262f8eb), for the S column + +- `crates/aprender-core/src/automl/`: AutoTuner, SearchSpace, TPE, GridSearch, RandomSearch, DESearch, ActiveLearningSearch, TimeBudget, EarlyStopping, ProgressCallback — tunes ONE chosen estimator. +- `model_selection/`: KFold, StratifiedKFold, cross_validate, cross_val_score, grid_search, randomized_search, train_test_split. +- `preprocessing/`: LabelEncoder, OneHotEncoder, OrdinalEncoder, Standard/MinMax/MaxAbs/Robust scalers, Normalizer, PolynomialFeatures, PCA, TSNE — all applied by hand, no type inference. +- `tree/`: DecisionTree{Classifier,Regressor}, RandomForest{Classifier,Regressor}, GradientBoostingClassifier (no regressor). +- `calibration.rs`: PlattScaling, IsotonicRegression, TemperatureScaling, ECE/MCE/Brier. No decision-threshold search. +- `interpret/`, `explainable/`: ShapExplainer, LIME, PermutationImportance, IntegratedGradients, CounterfactualExplainer — per estimator. +- `time_series/`: `ARIMA` only (fit/forecast/order), single f32 series. `metrics/`: no MASE/RMSSE/WQL. +- `ensemble/`: MixtureOfExperts + SoftmaxGating. `stack/`: deployment health, not model stacking. +- `apr train` = causal-LM pre-training; `apr finetune --task classify` = text classification. No `apr automl`, no `apr forecast`. diff --git a/evidence/crux/autogluon/hello.sh b/evidence/crux/autogluon/hello.sh new file mode 100644 index 0000000000..7b1662298a --- /dev/null +++ b/evidence/crux/autogluon/hello.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# AutoGluon canonical flow, transcribed from README.md + tabular quick start (1.6.3). +# Run with: uv run --with autogluon.tabular python - <<'PY' +set -euo pipefail +python - <<'PY' +from autogluon.tabular import TabularDataset, TabularPredictor +train = TabularDataset("https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv") +test = TabularDataset("https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv") +predictor = TabularPredictor(label="class").fit(train, presets="medium_quality", time_limit=120) +print(predictor.problem_type) # inferred: binary +print(predictor.leaderboard(test)) # model, score_test, score_val, pred_time_*, fit_time, stack_level +print(predictor.feature_importance(test)) # permutation importance on raw columns +predictor.clone_for_deployment("deploy/") # keep_only_best + save_space +PY diff --git a/evidence/crux/autogluon/readme-verbs.txt b/evidence/crux/autogluon/readme-verbs.txt new file mode 100644 index 0000000000..248b128e2b --- /dev/null +++ b/evidence/crux/autogluon/readme-verbs.txt @@ -0,0 +1,23 @@ +# AutoGluon 1.6.3 README verbs, ranked by fold position (../autogluon/README.md, 2026-09-16) +# fold 1 — the only code block in the README: +pip install autogluon +TabularPredictor(label="class").fit("train.csv", presets="best") +predictor.predict("test.csv") +# fold 2 — docs/tutorials/tabular/tabular-quick-start.ipynb call order: +TabularPredictor(label).fit(train_data, time_limit=...) +predictor.predict(test_data) +predictor.evaluate(test_data) +predictor.leaderboard(test_data) +predictor.feature_importance(test_data) +# fold 3 — timeseries quick start: +TimeSeriesPredictor(prediction_length=48, eval_metric="WQL").fit(TimeSeriesDataFrame, presets="medium_quality") +predictor.predict(train_data) +predictor.leaderboard(test_data) +# tabular presets (tabular/src/autogluon/tabular/configs/presets_configs.py): +extreme_quality best_quality high_quality good_quality medium_quality optimize_for_deployment ignore_text ignore_text_ngrams interpretable noncommercial tabarena +# tabular model families (tabular/src/autogluon/tabular/models/): +catboost ebm fastainn imodels knn lgb lr mitra nori realmlp rf tabdpt tabicl tabm tabpfnmix tabpfnv2 tabprep tabular_nn xgboost xt (+ automm/image/text wrappers) +# timeseries models (timeseries/src/autogluon/timeseries/models/): +Naive SeasonalNaive Average SeasonalAverage NPTS Zero | AutoARIMA ARIMA AutoETS ETS AutoCES DynamicOptimizedTheta Theta Croston ADIDA IMAPA | DeepAR SimpleFeedForward TemporalFusionTransformer DLinear PatchTST WaveNet TiDE | Chronos Chronos2 Toto Toto2 | PerStepTabular RecursiveTabular DirectTabular | ensembles: greedy selection, per-item greedy, weighted, array-based +# timeseries metrics (timeseries/src/autogluon/timeseries/metrics/): +MQL WQL SQL RMSE MSE MAE MAEB BIAS WAPE WAPEB SMAPE MAPE MASE RMSSE RMSLE WCD diff --git a/evidence/parity/EXPECTED_RECEIPTS b/evidence/parity/EXPECTED_RECEIPTS new file mode 100644 index 0000000000..370d2c0701 --- /dev/null +++ b/evidence/parity/EXPECTED_RECEIPTS @@ -0,0 +1,10 @@ +# PMAT-3577 / #3577 — the expected number of apr-parity-receipt/v2 focus nodes under evidence/parity/**. +# +# Produced by the committed predicate: bash scripts/parity_receipt_denominator.sh +# The extractor must reproduce this count exactly. A mismatch is Unknown{WrongCorpus}, exit 2 — never +# Pass, never a fabricated Fail. An extractor that silently matches fewer files than exist reports the same +# "no violations" as one that matches all of them, which is the vacuity hole this file closes. +# +# ADDING A RECEIPT? Update this number in the same commit. If you do not, the gate stops you — that is the +# falsifier working, not the gate being wrong. +7 diff --git a/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json index 19a6a0b884..dd79ff2c06 100644 --- a/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/gx10/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "21d182d69505159c", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1/gx10/RECORD.md heading '2026-09-08T12:5xZ' \u2014 hour stated, minute not", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9506108164787292, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "FAIL", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 15, - "gpu_argmax": 16, - "max_abs_diff": 12.008713722229004, - "mean_abs_diff": 1.4603080749511719, - "cosine_similarity": 0.9506108164787292, - "kl_divergence": 5.453976683901305, - "sigma_level": 2.138399453406798, - "cpk": 0.4525729853621727, - "verdict": "WarnOutOfSpec" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.4038071632385254, - "mean_abs_diff": 0.06531447172164917, - "cosine_similarity": 0.9998307228088379, - "kl_divergence": 0.00003169573582524447, - "sigma_level": 48.145552545660266, - "cpk": 15.786467404364531, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.47008025646209717, - "mean_abs_diff": 0.079141765832901, - "cosine_similarity": 0.9998854398727417, - "kl_divergence": 2.350304035683132e-6, - "sigma_level": 40.26894361164864, - "cpk": 13.157401595078596, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.34066465497016907, - "mean_abs_diff": 0.05701277032494545, - "cosine_similarity": 0.9998948574066162, - "kl_divergence": 0.00007930019129213315, - "sigma_level": 55.01750807501765, - "cpk": 18.0777776456949, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.36131715774536133, - "mean_abs_diff": 0.06654312461614609, - "cosine_similarity": 0.9998743534088135, - "kl_divergence": 5.586878639155519e-6, - "sigma_level": 48.151386331009974, - "cpk": 15.783450135247941, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.39881277084350586, - "mean_abs_diff": 0.05531281977891922, - "cosine_similarity": 0.9998670220375061, - "kl_divergence": 0.00012843890921633306, - "sigma_level": 56.67790489208867, - "cpk": 18.631383735800988, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.40886592864990234, - "mean_abs_diff": 0.06747263669967651, - "cosine_similarity": 0.9999094009399414, - "kl_divergence": 0.000011448242344448684, - "sigma_level": 47.16232087734927, - "cpk": 15.455593113910512, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.45634615421295166, - "mean_abs_diff": 0.06790852546691895, - "cosine_similarity": 0.9999128580093384, - "kl_divergence": 4.6069409166491056e-6, - "sigma_level": 46.829472258467334, - "cpk": 15.34481405203357, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3148663640022278, - "mean_abs_diff": 0.05503924563527107, - "cosine_similarity": 0.9998142719268799, - "kl_divergence": 0.0006964580484037847, - "sigma_level": 58.33280314138458, - "cpk": 19.17671825707048, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.37107133865356445, - "mean_abs_diff": 0.049693331122398376, - "cosine_similarity": 0.9998795390129089, - "kl_divergence": 0.0019487691332511425, - "sigma_level": 63.28510665986591, - "cpk": 20.832964906591553, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.3408195972442627, - "mean_abs_diff": 0.049113236367702484, - "cosine_similarity": 0.9999272227287292, - "kl_divergence": 0.0023197360102891846, - "sigma_level": 63.60800653625746, - "cpk": 20.942335923761355, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.2871994972229004, - "mean_abs_diff": 0.04618430510163307, - "cosine_similarity": 0.99989253282547, - "kl_divergence": 0.0010725620753604848, - "sigma_level": 67.48480073589947, - "cpk": 22.235205359724013, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.30663490295410156, - "mean_abs_diff": 0.05332659184932709, - "cosine_similarity": 0.9998827576637268, - "kl_divergence": 0.0003849714682721958, - "sigma_level": 60.14815823956778, - "cpk": 19.782094722778414, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.341217041015625, - "mean_abs_diff": 0.05129178240895271, - "cosine_similarity": 0.9998595714569092, - "kl_divergence": 0.0011121437754711468, - "sigma_level": 60.927599133008556, - "cpk": 20.048775947883673, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.3473668098449707, - "mean_abs_diff": 0.06534294039011002, - "cosine_similarity": 0.999890148639679, - "kl_divergence": 0.0018708287004877242, - "sigma_level": 50.02631084841449, - "cpk": 16.403031428829408, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3845231533050537, - "mean_abs_diff": 0.05668144300580025, - "cosine_similarity": 0.9998037815093994, - "kl_divergence": 0.0014187455405557119, - "sigma_level": 56.08514037244682, - "cpk": 18.43013123352451, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3365215063095093, - "mean_abs_diff": 0.051528360694646835, - "cosine_similarity": 0.999897837638855, - "kl_divergence": 0.001462163794542408, - "sigma_level": 61.80499140623471, - "cpk": 20.336271311252403, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.34671688079833984, - "mean_abs_diff": 0.05129155516624451, - "cosine_similarity": 0.999882161617279, - "kl_divergence": 0.00008450073376790388, - "sigma_level": 61.06263290998148, - "cpk": 20.09321118628562, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.3720208406448364, - "mean_abs_diff": 0.05188516154885292, - "cosine_similarity": 0.9998630881309509, - "kl_divergence": 0.0006878276376716025, - "sigma_level": 60.17461919193516, - "cpk": 19.79802557748553, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.364626407623291, - "mean_abs_diff": 0.077647864818573, - "cosine_similarity": 0.9997988343238831, - "kl_divergence": 0.004185557706680174, - "sigma_level": 43.092094054533995, - "cpk": 14.085197260353358, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.4366130828857422, - "mean_abs_diff": 0.0633498802781105, - "cosine_similarity": 0.9997689127922058, - "kl_divergence": 0.0032265442609676616, - "sigma_level": 49.67682516705651, - "cpk": 16.296689978441368, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 1.0816888809204102, - "mean_abs_diff": 0.16485518217086792, - "cosine_similarity": 0.9990662932395935, - "kl_divergence": 0.013121373568509257, - "sigma_level": 19.17278201676311, - "cpk": 6.1275329662463855, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.8328787088394165, - "mean_abs_diff": 0.12702322006225586, - "cosine_similarity": 0.9992517232894897, - "kl_divergence": 0.021413090149460762, - "sigma_level": 24.71860116353615, - "cpk": 7.977880694909801, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.8862452507019043, - "mean_abs_diff": 0.10550703853368759, - "cosine_similarity": 0.9994606971740723, - "kl_divergence": 0.0028384683547810416, - "sigma_level": 29.136462853420184, - "cpk": 9.455979125389131, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.4594208002090454, - "mean_abs_diff": 0.07261195033788681, - "cosine_similarity": 0.9997691512107849, - "kl_divergence": 0.0028756378761694195, - "sigma_level": 43.5227244869569, - "cpk": 14.244219003234265, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4376527667045593, - "mean_abs_diff": 0.04770943894982338, - "cosine_similarity": 0.9997802972793579, - "kl_divergence": 0.0002040249183579563, - "sigma_level": 65.76835287900975, - "cpk": 21.661303358293953, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3315706253051758, - "mean_abs_diff": 0.052924565970897675, - "cosine_similarity": 0.9998641014099121, - "kl_divergence": 0.0005738790430968991, - "sigma_level": 59.410746258878895, - "cpk": 19.54155808964644, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.4616411328315735, - "mean_abs_diff": 0.0820874571800232, - "cosine_similarity": 0.9993736147880554, - "kl_divergence": 0.000581956823405685, - "sigma_level": 39.33158269369345, - "cpk": 12.841475097048557, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.4140510559082031, - "mean_abs_diff": 0.08190098404884338, - "cosine_similarity": 0.9998055100440979, - "kl_divergence": 0.0045809146856644325, - "sigma_level": 39.888270533950774, - "cpk": 13.023849460588838, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.38747692108154297, - "mean_abs_diff": 0.06567694246768951, - "cosine_similarity": 0.9997859597206116, - "kl_divergence": 0.0007163080545155323, - "sigma_level": 48.645284717186506, - "cpk": 15.948855441920916, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.31238651275634766, - "mean_abs_diff": 0.046092480421066284, - "cosine_similarity": 0.9997801780700684, - "kl_divergence": 0.00007854547302267981, - "sigma_level": 67.973856818208, - "cpk": 22.396861967357868, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2587924003601074, - "mean_abs_diff": 0.04632105305790901, - "cosine_similarity": 0.9999061226844788, - "kl_divergence": 0.0006675378664987662, - "sigma_level": 67.8328996522191, - "cpk": 22.349125605417846, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.3423733711242676, - "mean_abs_diff": 0.0509999543428421, - "cosine_similarity": 0.9999075531959534, - "kl_divergence": 0.0013139015230752497, - "sigma_level": 61.44627485953518, - "cpk": 20.22094518548056, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 1075, - "max_abs_diff": 0.33937501907348633, - "mean_abs_diff": 0.05567406490445137, - "cosine_similarity": 0.9999013543128967, - "kl_divergence": 0.0023980062691288805, - "sigma_level": 57.02066163057764, - "cpk": 18.742339542149587, - "verdict": "WarnArgmax" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.30877065658569336, - "mean_abs_diff": 0.04503734037280083, - "cosine_similarity": 0.9998331665992737, - "kl_divergence": 0.00005757731281013704, - "sigma_level": 69.88200924330435, - "cpk": 23.031728094749294, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.34514331817626953, - "mean_abs_diff": 0.057505182921886444, - "cosine_similarity": 0.9999133348464966, - "kl_divergence": 0.0030512423213599734, - "sigma_level": 55.02250951493119, - "cpk": 18.07716321543728, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.39675191044807434, - "mean_abs_diff": 0.06584710627794266, - "cosine_similarity": 0.9995452761650085, - "kl_divergence": 0.002512817256532628, - "sigma_level": 47.60950042344511, - "cpk": 15.608587821629843, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.26343512535095215, - "mean_abs_diff": 0.04300215467810631, - "cosine_similarity": 0.9998367428779602, - "kl_divergence": 0.0007504716839459635, - "sigma_level": 73.06733630767575, - "cpk": 24.093941027740257, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.4165067672729492, - "mean_abs_diff": 0.0824187770485878, - "cosine_similarity": 0.9998924136161804, - "kl_divergence": 0.0009092099497840965, - "sigma_level": 40.712890472295875, - "cpk": 13.291337937195314, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.37496471405029297, - "mean_abs_diff": 0.07949145883321762, - "cosine_similarity": 0.9998467564582825, - "kl_divergence": 0.0021284045408820193, - "sigma_level": 42.300035778519224, - "cpk": 13.81980429677876, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.30150842666625977, - "mean_abs_diff": 0.05042034387588501, - "cosine_similarity": 0.9998810887336731, - "kl_divergence": 0.0015508871007383236, - "sigma_level": 61.98594278003023, - "cpk": 20.40153488080674, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.3162221908569336, - "mean_abs_diff": 0.050118133425712585, - "cosine_similarity": 0.9999067187309265, - "kl_divergence": 0.0020995741983847304, - "sigma_level": 63.78172992219015, - "cpk": 20.994191536533126, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.3547534942626953, - "mean_abs_diff": 0.05901937186717987, - "cosine_similarity": 0.9999095797538757, - "kl_divergence": 0.001992879373807011, - "sigma_level": 54.67646177583371, - "cpk": 17.956573056117104, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3257865905761719, - "mean_abs_diff": 0.04293311759829521, - "cosine_similarity": 0.9998363256454468, - "kl_divergence": 0.0003393573469818488, - "sigma_level": 72.82458829123628, - "cpk": 24.014313879315836, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3269984722137451, - "mean_abs_diff": 0.05209345370531082, - "cosine_similarity": 0.9998263120651245, - "kl_divergence": 0.0013256906000322828, - "sigma_level": 60.776719885309284, - "cpk": 19.995067524794262, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.3318147659301758, - "mean_abs_diff": 0.061590515077114105, - "cosine_similarity": 0.9999039769172668, - "kl_divergence": 0.0019106988855219238, - "sigma_level": 53.494702850667856, - "cpk": 17.557003758350135, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.2706027030944824, - "mean_abs_diff": 0.0453641451895237, - "cosine_similarity": 0.9998862147331238, - "kl_divergence": 0.0015863168518343554, - "sigma_level": 69.14568409477452, - "cpk": 22.787166793882815, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.320314884185791, - "mean_abs_diff": 0.05479338765144348, - "cosine_similarity": 0.9998805522918701, - "kl_divergence": 0.0018053862786531084, - "sigma_level": 58.55186723394565, - "cpk": 19.249934481393097, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.2487473487854004, - "mean_abs_diff": 0.04012826830148697, - "cosine_similarity": 0.9999148845672607, - "kl_divergence": 0.000992232432491818, - "sigma_level": 79.04261413932409, - "cpk": 26.083217777488557, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2852489948272705, - "mean_abs_diff": 0.04260895773768425, - "cosine_similarity": 0.9998908042907715, - "kl_divergence": 0.0007555272284797762, - "sigma_level": 74.2463000340896, - "cpk": 24.48513688966887, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.2984335422515869, - "mean_abs_diff": 0.044331666082143784, - "cosine_similarity": 0.9999169707298279, - "kl_divergence": 0.0029629589041888825, - "sigma_level": 70.46956643182976, - "cpk": 23.229519369942476, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 1030, - "max_abs_diff": 0.273104190826416, - "mean_abs_diff": 0.04725624620914459, - "cosine_similarity": 0.9999066591262817, - "kl_divergence": 0.0003615828044026549, - "sigma_level": 67.64386780596082, - "cpk": 22.281572996022163, - "verdict": "WarnArgmax" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.3086332678794861, - "mean_abs_diff": 0.04187058284878731, - "cosine_similarity": 0.9999241232872009, - "kl_divergence": 0.0016002433269929573, - "sigma_level": 74.5359265207267, - "cpk": 24.585236949692465, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2898261547088623, - "mean_abs_diff": 0.04877804219722748, - "cosine_similarity": 0.9998968243598938, - "kl_divergence": 0.0002295654035777167, - "sigma_level": 66.83978374060625, - "cpk": 22.008235097556014, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2525869607925415, - "mean_abs_diff": 0.038769688457250595, - "cosine_similarity": 0.9998642802238464, - "kl_divergence": 0.00015459832954484046, - "sigma_level": 81.89698771623543, - "cpk": 27.03440251379967, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.2620408535003662, - "mean_abs_diff": 0.038319140672683716, - "cosine_similarity": 0.9998922944068909, - "kl_divergence": 0.0003140919129558196, - "sigma_level": 82.67792301311985, - "cpk": 27.29529542416786, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.3415346145629883, - "mean_abs_diff": 0.07152623683214188, - "cosine_similarity": 0.9998765587806702, - "kl_divergence": 0.0012743173908936454, - "sigma_level": 47.2490865840439, - "cpk": 15.46806641492191, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.30021238327026367, - "mean_abs_diff": 0.050972770899534225, - "cosine_similarity": 0.9998793601989746, - "kl_divergence": 0.0007696301505989618, - "sigma_level": 63.01291429802815, - "cpk": 20.736642862323936, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.2957894802093506, - "mean_abs_diff": 0.042167410254478455, - "cosine_similarity": 0.9998956322669983, - "kl_divergence": 0.0000986269813444574, - "sigma_level": 74.45760637654595, - "cpk": 24.557561755961462, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.24124550819396973, - "mean_abs_diff": 0.040473658591508865, - "cosine_similarity": 0.9999110102653503, - "kl_divergence": 0.0005439255887544281, - "sigma_level": 77.81332168413442, - "cpk": 25.675324743401898, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.2903571128845215, - "mean_abs_diff": 0.05764927715063095, - "cosine_similarity": 0.9999284148216248, - "kl_divergence": 0.0006913008282435065, - "sigma_level": 56.68813858123516, - "cpk": 18.62371034272647, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2806577682495117, - "mean_abs_diff": 0.04559638351202011, - "cosine_similarity": 0.9998930096626282, - "kl_divergence": 0.00037460736880700403, - "sigma_level": 69.88090861817952, - "cpk": 23.028109813599595, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2667236328125, - "mean_abs_diff": 0.043514035642147064, - "cosine_similarity": 0.9999178051948547, - "kl_divergence": 0.0021676753013869826, - "sigma_level": 72.52037518933956, - "cpk": 23.910487213882288, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2771492004394531, - "mean_abs_diff": 0.0458214171230793, - "cosine_similarity": 0.9998854398727417, - "kl_divergence": 0.0005921675492852606, - "sigma_level": 69.37134539440211, - "cpk": 22.858890685325196, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2921719551086426, - "mean_abs_diff": 0.04640787094831467, - "cosine_similarity": 0.9999167919158936, - "kl_divergence": 0.0010064557477895723, - "sigma_level": 68.66264823251565, - "cpk": 22.62200880099321, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.2976968288421631, - "mean_abs_diff": 0.043953705579042435, - "cosine_similarity": 0.9999005198478699, - "kl_divergence": 0.002171449635300438, - "sigma_level": 71.44745250213712, - "cpk": 23.554119143074743, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23976421356201172, - "mean_abs_diff": 0.041163649410009384, - "cosine_similarity": 0.9998940229415894, - "kl_divergence": 0.00028813644654819, - "sigma_level": 76.62883812597585, - "cpk": 25.280085823049113, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.28889644145965576, - "mean_abs_diff": 0.05656943470239639, - "cosine_similarity": 0.9998864531517029, - "kl_divergence": 0.0014977009727720507, - "sigma_level": 57.23183046776684, - "cpk": 18.8074791312102, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.29786229133605957, - "mean_abs_diff": 0.049185607582330704, - "cosine_similarity": 0.9998613595962524, - "kl_divergence": 0.00020494337876287373, - "sigma_level": 64.53610010140734, - "cpk": 21.247512759262296, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.3566131591796875, - "mean_abs_diff": 0.04558814316987991, - "cosine_similarity": 0.999882161617279, - "kl_divergence": 0.000034609400973773357, - "sigma_level": 68.88439368954486, - "cpk": 22.69977192970751, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.27669310569763184, - "mean_abs_diff": 0.041648074984550476, - "cosine_similarity": 0.9999054074287415, - "kl_divergence": 0.00014012989035636092, - "sigma_level": 75.50820301344545, - "cpk": 24.907336729394096, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.28789544105529785, - "mean_abs_diff": 0.042276348918676376, - "cosine_similarity": 0.9999191164970398, - "kl_divergence": 0.00047324870857447856, - "sigma_level": 74.10473544425255, - "cpk": 24.440505343736902, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.28496575355529785, - "mean_abs_diff": 0.05814102292060852, - "cosine_similarity": 0.999931812286377, - "kl_divergence": 0.0006591423793313406, - "sigma_level": 56.58944499601091, - "cpk": 18.58896764712217, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.27792835235595703, - "mean_abs_diff": 0.04199598729610443, - "cosine_similarity": 0.9999072551727295, - "kl_divergence": 0.001638797341847131, - "sigma_level": 74.50897424492256, - "cpk": 24.57556825365456, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.3433370590209961, - "mean_abs_diff": 0.053227003663778305, - "cosine_similarity": 0.9998949766159058, - "kl_divergence": 0.000473337168661806, - "sigma_level": 59.86797465225015, - "cpk": 19.69044214190352, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 1865, - "gpu_argmax": 6509, - "max_abs_diff": 0.3208746016025543, - "mean_abs_diff": 0.04410288855433464, - "cosine_similarity": 0.9999158382415771, - "kl_divergence": 0.0011789868514640338, - "sigma_level": 70.86618468274246, - "cpk": 23.361611273802996, - "verdict": "WarnArgmax" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.24564409255981445, - "mean_abs_diff": 0.04126206785440445, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.00040468322607162304, - "sigma_level": 75.96218013223144, - "cpk": 25.05953032482843, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.2946641445159912, - "mean_abs_diff": 0.04794362932443619, - "cosine_similarity": 0.9998166561126709, - "kl_divergence": 0.0009396785610877979, - "sigma_level": 65.26452919527235, - "cpk": 21.494091532109785, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 15, + "gpu_argmax": 16, + "max_abs_diff": 12.008713722229004, + "mean_abs_diff": 1.4603080749511719, + "cosine_similarity": 0.9506108164787292, + "kl_divergence": 5.453976683901305, + "sigma_level": 2.138399453406798, + "cpk": 0.4525729853621727, + "verdict": "WarnOutOfSpec" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.4038071632385254, + "mean_abs_diff": 0.06531447172164917, + "cosine_similarity": 0.9998307228088379, + "kl_divergence": 3.169573582524447e-05, + "sigma_level": 48.145552545660266, + "cpk": 15.786467404364531, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.47008025646209717, + "mean_abs_diff": 0.079141765832901, + "cosine_similarity": 0.9998854398727417, + "kl_divergence": 2.350304035683132e-06, + "sigma_level": 40.26894361164864, + "cpk": 13.157401595078596, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.34066465497016907, + "mean_abs_diff": 0.05701277032494545, + "cosine_similarity": 0.9998948574066162, + "kl_divergence": 7.930019129213315e-05, + "sigma_level": 55.01750807501765, + "cpk": 18.0777776456949, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.36131715774536133, + "mean_abs_diff": 0.06654312461614609, + "cosine_similarity": 0.9998743534088135, + "kl_divergence": 5.586878639155519e-06, + "sigma_level": 48.151386331009974, + "cpk": 15.783450135247941, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.39881277084350586, + "mean_abs_diff": 0.05531281977891922, + "cosine_similarity": 0.9998670220375061, + "kl_divergence": 0.00012843890921633306, + "sigma_level": 56.67790489208867, + "cpk": 18.631383735800988, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.40886592864990234, + "mean_abs_diff": 0.06747263669967651, + "cosine_similarity": 0.9999094009399414, + "kl_divergence": 1.1448242344448684e-05, + "sigma_level": 47.16232087734927, + "cpk": 15.455593113910512, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.45634615421295166, + "mean_abs_diff": 0.06790852546691895, + "cosine_similarity": 0.9999128580093384, + "kl_divergence": 4.6069409166491056e-06, + "sigma_level": 46.829472258467334, + "cpk": 15.34481405203357, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3148663640022278, + "mean_abs_diff": 0.05503924563527107, + "cosine_similarity": 0.9998142719268799, + "kl_divergence": 0.0006964580484037847, + "sigma_level": 58.33280314138458, + "cpk": 19.17671825707048, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.37107133865356445, + "mean_abs_diff": 0.049693331122398376, + "cosine_similarity": 0.9998795390129089, + "kl_divergence": 0.0019487691332511425, + "sigma_level": 63.28510665986591, + "cpk": 20.832964906591553, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.3408195972442627, + "mean_abs_diff": 0.049113236367702484, + "cosine_similarity": 0.9999272227287292, + "kl_divergence": 0.0023197360102891846, + "sigma_level": 63.60800653625746, + "cpk": 20.942335923761355, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.2871994972229004, + "mean_abs_diff": 0.04618430510163307, + "cosine_similarity": 0.99989253282547, + "kl_divergence": 0.0010725620753604848, + "sigma_level": 67.48480073589947, + "cpk": 22.235205359724013, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.30663490295410156, + "mean_abs_diff": 0.05332659184932709, + "cosine_similarity": 0.9998827576637268, + "kl_divergence": 0.0003849714682721958, + "sigma_level": 60.14815823956778, + "cpk": 19.782094722778414, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.341217041015625, + "mean_abs_diff": 0.05129178240895271, + "cosine_similarity": 0.9998595714569092, + "kl_divergence": 0.0011121437754711468, + "sigma_level": 60.927599133008556, + "cpk": 20.048775947883673, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.3473668098449707, + "mean_abs_diff": 0.06534294039011002, + "cosine_similarity": 0.999890148639679, + "kl_divergence": 0.0018708287004877242, + "sigma_level": 50.02631084841449, + "cpk": 16.403031428829408, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3845231533050537, + "mean_abs_diff": 0.05668144300580025, + "cosine_similarity": 0.9998037815093994, + "kl_divergence": 0.0014187455405557119, + "sigma_level": 56.08514037244682, + "cpk": 18.43013123352451, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3365215063095093, + "mean_abs_diff": 0.051528360694646835, + "cosine_similarity": 0.999897837638855, + "kl_divergence": 0.001462163794542408, + "sigma_level": 61.80499140623471, + "cpk": 20.336271311252403, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.34671688079833984, + "mean_abs_diff": 0.05129155516624451, + "cosine_similarity": 0.999882161617279, + "kl_divergence": 8.450073376790388e-05, + "sigma_level": 61.06263290998148, + "cpk": 20.09321118628562, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.3720208406448364, + "mean_abs_diff": 0.05188516154885292, + "cosine_similarity": 0.9998630881309509, + "kl_divergence": 0.0006878276376716025, + "sigma_level": 60.17461919193516, + "cpk": 19.79802557748553, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.364626407623291, + "mean_abs_diff": 0.077647864818573, + "cosine_similarity": 0.9997988343238831, + "kl_divergence": 0.004185557706680174, + "sigma_level": 43.092094054533995, + "cpk": 14.085197260353358, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.4366130828857422, + "mean_abs_diff": 0.0633498802781105, + "cosine_similarity": 0.9997689127922058, + "kl_divergence": 0.0032265442609676616, + "sigma_level": 49.67682516705651, + "cpk": 16.296689978441368, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 1.0816888809204102, + "mean_abs_diff": 0.16485518217086792, + "cosine_similarity": 0.9990662932395935, + "kl_divergence": 0.013121373568509257, + "sigma_level": 19.17278201676311, + "cpk": 6.1275329662463855, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.8328787088394165, + "mean_abs_diff": 0.12702322006225586, + "cosine_similarity": 0.9992517232894897, + "kl_divergence": 0.021413090149460762, + "sigma_level": 24.71860116353615, + "cpk": 7.977880694909801, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.8862452507019043, + "mean_abs_diff": 0.10550703853368759, + "cosine_similarity": 0.9994606971740723, + "kl_divergence": 0.0028384683547810416, + "sigma_level": 29.136462853420184, + "cpk": 9.455979125389131, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.4594208002090454, + "mean_abs_diff": 0.07261195033788681, + "cosine_similarity": 0.9997691512107849, + "kl_divergence": 0.0028756378761694195, + "sigma_level": 43.5227244869569, + "cpk": 14.244219003234265, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4376527667045593, + "mean_abs_diff": 0.04770943894982338, + "cosine_similarity": 0.9997802972793579, + "kl_divergence": 0.0002040249183579563, + "sigma_level": 65.76835287900975, + "cpk": 21.661303358293953, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3315706253051758, + "mean_abs_diff": 0.052924565970897675, + "cosine_similarity": 0.9998641014099121, + "kl_divergence": 0.0005738790430968991, + "sigma_level": 59.410746258878895, + "cpk": 19.54155808964644, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.4616411328315735, + "mean_abs_diff": 0.0820874571800232, + "cosine_similarity": 0.9993736147880554, + "kl_divergence": 0.000581956823405685, + "sigma_level": 39.33158269369345, + "cpk": 12.841475097048557, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.4140510559082031, + "mean_abs_diff": 0.08190098404884338, + "cosine_similarity": 0.9998055100440979, + "kl_divergence": 0.0045809146856644325, + "sigma_level": 39.888270533950774, + "cpk": 13.023849460588838, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.38747692108154297, + "mean_abs_diff": 0.06567694246768951, + "cosine_similarity": 0.9997859597206116, + "kl_divergence": 0.0007163080545155323, + "sigma_level": 48.645284717186506, + "cpk": 15.948855441920916, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.31238651275634766, + "mean_abs_diff": 0.046092480421066284, + "cosine_similarity": 0.9997801780700684, + "kl_divergence": 7.854547302267981e-05, + "sigma_level": 67.973856818208, + "cpk": 22.396861967357868, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2587924003601074, + "mean_abs_diff": 0.04632105305790901, + "cosine_similarity": 0.9999061226844788, + "kl_divergence": 0.0006675378664987662, + "sigma_level": 67.8328996522191, + "cpk": 22.349125605417846, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.3423733711242676, + "mean_abs_diff": 0.0509999543428421, + "cosine_similarity": 0.9999075531959534, + "kl_divergence": 0.0013139015230752497, + "sigma_level": 61.44627485953518, + "cpk": 20.22094518548056, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 1075, + "max_abs_diff": 0.33937501907348633, + "mean_abs_diff": 0.05567406490445137, + "cosine_similarity": 0.9999013543128967, + "kl_divergence": 0.0023980062691288805, + "sigma_level": 57.02066163057764, + "cpk": 18.742339542149587, + "verdict": "WarnArgmax" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.30877065658569336, + "mean_abs_diff": 0.04503734037280083, + "cosine_similarity": 0.9998331665992737, + "kl_divergence": 5.757731281013704e-05, + "sigma_level": 69.88200924330435, + "cpk": 23.031728094749294, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.34514331817626953, + "mean_abs_diff": 0.057505182921886444, + "cosine_similarity": 0.9999133348464966, + "kl_divergence": 0.0030512423213599734, + "sigma_level": 55.02250951493119, + "cpk": 18.07716321543728, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.39675191044807434, + "mean_abs_diff": 0.06584710627794266, + "cosine_similarity": 0.9995452761650085, + "kl_divergence": 0.002512817256532628, + "sigma_level": 47.60950042344511, + "cpk": 15.608587821629843, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.26343512535095215, + "mean_abs_diff": 0.04300215467810631, + "cosine_similarity": 0.9998367428779602, + "kl_divergence": 0.0007504716839459635, + "sigma_level": 73.06733630767575, + "cpk": 24.093941027740257, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.4165067672729492, + "mean_abs_diff": 0.0824187770485878, + "cosine_similarity": 0.9998924136161804, + "kl_divergence": 0.0009092099497840965, + "sigma_level": 40.712890472295875, + "cpk": 13.291337937195314, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.37496471405029297, + "mean_abs_diff": 0.07949145883321762, + "cosine_similarity": 0.9998467564582825, + "kl_divergence": 0.0021284045408820193, + "sigma_level": 42.300035778519224, + "cpk": 13.81980429677876, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.30150842666625977, + "mean_abs_diff": 0.05042034387588501, + "cosine_similarity": 0.9998810887336731, + "kl_divergence": 0.0015508871007383236, + "sigma_level": 61.98594278003023, + "cpk": 20.40153488080674, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.3162221908569336, + "mean_abs_diff": 0.050118133425712585, + "cosine_similarity": 0.9999067187309265, + "kl_divergence": 0.0020995741983847304, + "sigma_level": 63.78172992219015, + "cpk": 20.994191536533126, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.3547534942626953, + "mean_abs_diff": 0.05901937186717987, + "cosine_similarity": 0.9999095797538757, + "kl_divergence": 0.001992879373807011, + "sigma_level": 54.67646177583371, + "cpk": 17.956573056117104, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3257865905761719, + "mean_abs_diff": 0.04293311759829521, + "cosine_similarity": 0.9998363256454468, + "kl_divergence": 0.0003393573469818488, + "sigma_level": 72.82458829123628, + "cpk": 24.014313879315836, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3269984722137451, + "mean_abs_diff": 0.05209345370531082, + "cosine_similarity": 0.9998263120651245, + "kl_divergence": 0.0013256906000322828, + "sigma_level": 60.776719885309284, + "cpk": 19.995067524794262, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.3318147659301758, + "mean_abs_diff": 0.061590515077114105, + "cosine_similarity": 0.9999039769172668, + "kl_divergence": 0.0019106988855219238, + "sigma_level": 53.494702850667856, + "cpk": 17.557003758350135, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.2706027030944824, + "mean_abs_diff": 0.0453641451895237, + "cosine_similarity": 0.9998862147331238, + "kl_divergence": 0.0015863168518343554, + "sigma_level": 69.14568409477452, + "cpk": 22.787166793882815, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.320314884185791, + "mean_abs_diff": 0.05479338765144348, + "cosine_similarity": 0.9998805522918701, + "kl_divergence": 0.0018053862786531084, + "sigma_level": 58.55186723394565, + "cpk": 19.249934481393097, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.2487473487854004, + "mean_abs_diff": 0.04012826830148697, + "cosine_similarity": 0.9999148845672607, + "kl_divergence": 0.000992232432491818, + "sigma_level": 79.04261413932409, + "cpk": 26.083217777488557, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2852489948272705, + "mean_abs_diff": 0.04260895773768425, + "cosine_similarity": 0.9998908042907715, + "kl_divergence": 0.0007555272284797762, + "sigma_level": 74.2463000340896, + "cpk": 24.48513688966887, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.2984335422515869, + "mean_abs_diff": 0.044331666082143784, + "cosine_similarity": 0.9999169707298279, + "kl_divergence": 0.0029629589041888825, + "sigma_level": 70.46956643182976, + "cpk": 23.229519369942476, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 1030, + "max_abs_diff": 0.273104190826416, + "mean_abs_diff": 0.04725624620914459, + "cosine_similarity": 0.9999066591262817, + "kl_divergence": 0.0003615828044026549, + "sigma_level": 67.64386780596082, + "cpk": 22.281572996022163, + "verdict": "WarnArgmax" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.3086332678794861, + "mean_abs_diff": 0.04187058284878731, + "cosine_similarity": 0.9999241232872009, + "kl_divergence": 0.0016002433269929573, + "sigma_level": 74.5359265207267, + "cpk": 24.585236949692465, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2898261547088623, + "mean_abs_diff": 0.04877804219722748, + "cosine_similarity": 0.9998968243598938, + "kl_divergence": 0.0002295654035777167, + "sigma_level": 66.83978374060625, + "cpk": 22.008235097556014, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2525869607925415, + "mean_abs_diff": 0.038769688457250595, + "cosine_similarity": 0.9998642802238464, + "kl_divergence": 0.00015459832954484046, + "sigma_level": 81.89698771623543, + "cpk": 27.03440251379967, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.2620408535003662, + "mean_abs_diff": 0.038319140672683716, + "cosine_similarity": 0.9998922944068909, + "kl_divergence": 0.0003140919129558196, + "sigma_level": 82.67792301311985, + "cpk": 27.29529542416786, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.3415346145629883, + "mean_abs_diff": 0.07152623683214188, + "cosine_similarity": 0.9998765587806702, + "kl_divergence": 0.0012743173908936454, + "sigma_level": 47.2490865840439, + "cpk": 15.46806641492191, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.30021238327026367, + "mean_abs_diff": 0.050972770899534225, + "cosine_similarity": 0.9998793601989746, + "kl_divergence": 0.0007696301505989618, + "sigma_level": 63.01291429802815, + "cpk": 20.736642862323936, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.2957894802093506, + "mean_abs_diff": 0.042167410254478455, + "cosine_similarity": 0.9998956322669983, + "kl_divergence": 9.86269813444574e-05, + "sigma_level": 74.45760637654595, + "cpk": 24.557561755961462, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.24124550819396973, + "mean_abs_diff": 0.040473658591508865, + "cosine_similarity": 0.9999110102653503, + "kl_divergence": 0.0005439255887544281, + "sigma_level": 77.81332168413442, + "cpk": 25.675324743401898, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.2903571128845215, + "mean_abs_diff": 0.05764927715063095, + "cosine_similarity": 0.9999284148216248, + "kl_divergence": 0.0006913008282435065, + "sigma_level": 56.68813858123516, + "cpk": 18.62371034272647, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2806577682495117, + "mean_abs_diff": 0.04559638351202011, + "cosine_similarity": 0.9998930096626282, + "kl_divergence": 0.00037460736880700403, + "sigma_level": 69.88090861817952, + "cpk": 23.028109813599595, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2667236328125, + "mean_abs_diff": 0.043514035642147064, + "cosine_similarity": 0.9999178051948547, + "kl_divergence": 0.0021676753013869826, + "sigma_level": 72.52037518933956, + "cpk": 23.910487213882288, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2771492004394531, + "mean_abs_diff": 0.0458214171230793, + "cosine_similarity": 0.9998854398727417, + "kl_divergence": 0.0005921675492852606, + "sigma_level": 69.37134539440211, + "cpk": 22.858890685325196, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2921719551086426, + "mean_abs_diff": 0.04640787094831467, + "cosine_similarity": 0.9999167919158936, + "kl_divergence": 0.0010064557477895723, + "sigma_level": 68.66264823251565, + "cpk": 22.62200880099321, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.2976968288421631, + "mean_abs_diff": 0.043953705579042435, + "cosine_similarity": 0.9999005198478699, + "kl_divergence": 0.002171449635300438, + "sigma_level": 71.44745250213712, + "cpk": 23.554119143074743, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23976421356201172, + "mean_abs_diff": 0.041163649410009384, + "cosine_similarity": 0.9998940229415894, + "kl_divergence": 0.00028813644654819, + "sigma_level": 76.62883812597585, + "cpk": 25.280085823049113, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.28889644145965576, + "mean_abs_diff": 0.05656943470239639, + "cosine_similarity": 0.9998864531517029, + "kl_divergence": 0.0014977009727720507, + "sigma_level": 57.23183046776684, + "cpk": 18.8074791312102, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.29786229133605957, + "mean_abs_diff": 0.049185607582330704, + "cosine_similarity": 0.9998613595962524, + "kl_divergence": 0.00020494337876287373, + "sigma_level": 64.53610010140734, + "cpk": 21.247512759262296, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.3566131591796875, + "mean_abs_diff": 0.04558814316987991, + "cosine_similarity": 0.999882161617279, + "kl_divergence": 3.4609400973773357e-05, + "sigma_level": 68.88439368954486, + "cpk": 22.69977192970751, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.27669310569763184, + "mean_abs_diff": 0.041648074984550476, + "cosine_similarity": 0.9999054074287415, + "kl_divergence": 0.00014012989035636092, + "sigma_level": 75.50820301344545, + "cpk": 24.907336729394096, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.28789544105529785, + "mean_abs_diff": 0.042276348918676376, + "cosine_similarity": 0.9999191164970398, + "kl_divergence": 0.00047324870857447856, + "sigma_level": 74.10473544425255, + "cpk": 24.440505343736902, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.28496575355529785, + "mean_abs_diff": 0.05814102292060852, + "cosine_similarity": 0.999931812286377, + "kl_divergence": 0.0006591423793313406, + "sigma_level": 56.58944499601091, + "cpk": 18.58896764712217, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.27792835235595703, + "mean_abs_diff": 0.04199598729610443, + "cosine_similarity": 0.9999072551727295, + "kl_divergence": 0.001638797341847131, + "sigma_level": 74.50897424492256, + "cpk": 24.57556825365456, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.3433370590209961, + "mean_abs_diff": 0.053227003663778305, + "cosine_similarity": 0.9998949766159058, + "kl_divergence": 0.000473337168661806, + "sigma_level": 59.86797465225015, + "cpk": 19.69044214190352, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 1865, + "gpu_argmax": 6509, + "max_abs_diff": 0.3208746016025543, + "mean_abs_diff": 0.04410288855433464, + "cosine_similarity": 0.9999158382415771, + "kl_divergence": 0.0011789868514640338, + "sigma_level": 70.86618468274246, + "cpk": 23.361611273802996, + "verdict": "WarnArgmax" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.24564409255981445, + "mean_abs_diff": 0.04126206785440445, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.00040468322607162304, + "sigma_level": 75.96218013223144, + "cpk": 25.05953032482843, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.2946641445159912, + "mean_abs_diff": 0.04794362932443619, + "cosine_similarity": 0.9998166561126709, + "kl_divergence": 0.0009396785610877979, + "sigma_level": 65.26452919527235, + "cpk": 21.494091532109785, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json index a855176590..094462a85e 100644 --- a/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-08", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-08. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/gx10/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "21d182d69505159c", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1/gx10/RECORD.md heading '2026-09-08T12:5xZ' \u2014 hour stated, minute not", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9984647035598755, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.8031983375549316, - "mean_abs_diff": 0.12188125401735306, - "cosine_similarity": 0.9984647035598755, - "kl_divergence": 0.011949143213562554, - "sigma_level": 26.330537211297408, - "cpk": 8.50941249591051, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.24585700035095215, - "mean_abs_diff": 0.03762512281537056, - "cosine_similarity": 0.9998586177825928, - "kl_divergence": 0.000013277331549664009, - "sigma_level": 84.0184221427533, - "cpk": 27.742707093261536, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.2099609375, - "mean_abs_diff": 0.033425211906433105, - "cosine_similarity": 0.9999147653579712, - "kl_divergence": 1.5090158264594303e-6, - "sigma_level": 94.57560609882762, - "cpk": 31.261767893356485, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.22704386711120605, - "mean_abs_diff": 0.0315629281103611, - "cosine_similarity": 0.999933660030365, - "kl_divergence": 0.000037310540638633336, - "sigma_level": 99.18469092152263, - "cpk": 32.80068370140717, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.2088146209716797, - "mean_abs_diff": 0.03077378123998642, - "cosine_similarity": 0.9999191164970398, - "kl_divergence": 1.4999781410245668e-6, - "sigma_level": 101.59603971600392, - "cpk": 33.60480538024553, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2547646760940552, - "mean_abs_diff": 0.03083600476384163, - "cosine_similarity": 0.9999075531959534, - "kl_divergence": 0.0002142816112921191, - "sigma_level": 101.00436248100775, - "cpk": 33.40857324344982, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.20438094437122345, - "mean_abs_diff": 0.031773317605257034, - "cosine_similarity": 0.9999481439590454, - "kl_divergence": 9.281385733172737e-7, - "sigma_level": 99.44615585236863, - "cpk": 32.88540742624628, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.18281269073486328, - "mean_abs_diff": 0.030860792845487595, - "cosine_similarity": 0.9999558925628662, - "kl_divergence": 2.0817027272510226e-6, - "sigma_level": 102.06207588666598, - "cpk": 33.75821558044542, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.28357791900634766, - "mean_abs_diff": 0.03390377387404442, - "cosine_similarity": 0.999890148639679, - "kl_divergence": 0.00045742262385036316, - "sigma_level": 92.77850864421463, - "cpk": 30.6640410832845, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2932462692260742, - "mean_abs_diff": 0.044488128274679184, - "cosine_similarity": 0.9998958706855774, - "kl_divergence": 0.0016326477262920424, - "sigma_level": 71.4951399913799, - "cpk": 23.566656250547247, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.25107574462890625, - "mean_abs_diff": 0.036512766033411026, - "cosine_similarity": 0.9999284148216248, - "kl_divergence": 0.0011823933043749269, - "sigma_level": 85.6818989695192, - "cpk": 28.299926062308696, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.25175952911376953, - "mean_abs_diff": 0.04251187667250633, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.000520927378713784, - "sigma_level": 75.90532316155377, - "cpk": 25.032867909098698, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.23739862442016602, - "mean_abs_diff": 0.035071682184934616, - "cosine_similarity": 0.9999188184738159, - "kl_divergence": 0.00016192173524096987, - "sigma_level": 91.2163777694204, - "cpk": 30.13886660554096, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.20715034008026123, - "mean_abs_diff": 0.031439900398254395, - "cosine_similarity": 0.9999179244041443, - "kl_divergence": 0.0007082983479816206, - "sigma_level": 100.4901828181864, - "cpk": 33.23344416116162, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.23068474233150482, - "mean_abs_diff": 0.03183136135339737, - "cosine_similarity": 0.9999409914016724, - "kl_divergence": 0.0007547449349859649, - "sigma_level": 98.39571148132515, - "cpk": 32.53756470645949, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23305463790893555, - "mean_abs_diff": 0.028933987021446228, - "cosine_similarity": 0.9998893141746521, - "kl_divergence": 0.0003845604489353084, - "sigma_level": 107.47455288493924, - "cpk": 35.565712018454036, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2608985900878906, - "mean_abs_diff": 0.04424131289124489, - "cosine_similarity": 0.9999105930328369, - "kl_divergence": 0.0007788479711618704, - "sigma_level": 73.24530032745929, - "cpk": 24.14506108835307, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.2301793098449707, - "mean_abs_diff": 0.0324413925409317, - "cosine_similarity": 0.9999143481254578, - "kl_divergence": 0.00002250089535157956, - "sigma_level": 96.25744820928855, - "cpk": 31.825588931234023, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.19923532009124756, - "mean_abs_diff": 0.03906163200736046, - "cosine_similarity": 0.999877393245697, - "kl_divergence": 0.0005163370630241004, - "sigma_level": 83.7633797367138, - "cpk": 27.648465385990573, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 5942, - "gpu_argmax": 5942, - "max_abs_diff": 0.17966842651367188, - "mean_abs_diff": 0.02848205156624317, - "cosine_similarity": 0.9999198317527771, - "kl_divergence": 0.0008812988738490747, - "sigma_level": 109.6664749254824, - "cpk": 36.29519779233449, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.29142284393310547, - "mean_abs_diff": 0.043598152697086334, - "cosine_similarity": 0.9998776316642761, - "kl_divergence": 0.0010248388343692954, - "sigma_level": 73.4393195777277, - "cpk": 24.21295497016591, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.34327876567840576, - "mean_abs_diff": 0.0488663986325264, - "cosine_similarity": 0.9999039173126221, - "kl_divergence": 0.0018092925711578334, - "sigma_level": 64.72170503736106, - "cpk": 21.310341959242646, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3739614486694336, - "mean_abs_diff": 0.04563745856285095, - "cosine_similarity": 0.9998410940170288, - "kl_divergence": 0.0013028657116986625, - "sigma_level": 68.52047136061793, - "cpk": 22.579565439170377, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.25923848152160645, - "mean_abs_diff": 0.04782666265964508, - "cosine_similarity": 0.9998435378074646, - "kl_divergence": 0.00017440859928140556, - "sigma_level": 68.0917989073902, - "cpk": 22.425882677777388, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.20351624488830566, - "mean_abs_diff": 0.030075596645474434, - "cosine_similarity": 0.9998764991760254, - "kl_divergence": 0.0005101532884886966, - "sigma_level": 103.98703355346954, - "cpk": 34.40172184469705, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.20961999893188477, - "mean_abs_diff": 0.0313001312315464, - "cosine_similarity": 0.9998053312301636, - "kl_divergence": 0.000028302109163839864, - "sigma_level": 102.59202935793, - "cpk": 33.929747787458844, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19363737106323242, - "mean_abs_diff": 0.033073790371418, - "cosine_similarity": 0.9998298287391663, - "kl_divergence": 0.0005171189703267121, - "sigma_level": 96.9427542695765, - "cpk": 32.04706272879717, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.20902681350708008, - "mean_abs_diff": 0.03569749742746353, - "cosine_similarity": 0.9997431039810181, - "kl_divergence": 0.0001146219937046406, - "sigma_level": 90.59358669300643, - "cpk": 29.92836520367562, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.20511221885681152, - "mean_abs_diff": 0.03204226866364479, - "cosine_similarity": 0.9998698830604553, - "kl_divergence": 0.0009168103274381441, - "sigma_level": 97.91782700745156, - "cpk": 32.3778165591561, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.20564067363739014, - "mean_abs_diff": 0.03282441571354866, - "cosine_similarity": 0.9998794794082642, - "kl_divergence": 0.00003538160864634364, - "sigma_level": 96.38664467254054, - "cpk": 31.865228616349718, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2244204878807068, - "mean_abs_diff": 0.048710912466049194, - "cosine_similarity": 0.9996637105941772, - "kl_divergence": 3.898172129763024e-6, - "sigma_level": 69.19204623397793, - "cpk": 22.7831481023718, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19256067276000977, - "mean_abs_diff": 0.038800157606601715, - "cosine_similarity": 0.9998534321784973, - "kl_divergence": 0.00014612402448690114, - "sigma_level": 83.81763111733832, - "cpk": 27.668198930982378, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.24395322799682617, - "mean_abs_diff": 0.035089071840047836, - "cosine_similarity": 0.9998824596405029, - "kl_divergence": 0.0007183325325718559, - "sigma_level": 90.07186375980564, - "cpk": 29.76057641174898, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.22237825393676758, - "mean_abs_diff": 0.030163198709487915, - "cosine_similarity": 0.9998973608016968, - "kl_divergence": 0.0007083169726316857, - "sigma_level": 103.65308835461354, - "cpk": 34.290487059796824, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.19781363010406494, - "mean_abs_diff": 0.028445789590477943, - "cosine_similarity": 0.9998837113380432, - "kl_divergence": 9.986173050612662e-6, - "sigma_level": 109.84427252512704, - "cpk": 36.35437358637827, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.21171927452087402, - "mean_abs_diff": 0.0331043042242527, - "cosine_similarity": 0.999922513961792, - "kl_divergence": 0.0005099449753999734, - "sigma_level": 95.18447747645524, - "cpk": 31.465574500501155, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.2000875473022461, - "mean_abs_diff": 0.028575299307703972, - "cosine_similarity": 0.9998475313186646, - "kl_divergence": 0.001018203258176856, - "sigma_level": 109.12304284617422, - "cpk": 36.11449564783334, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.18700814247131348, - "mean_abs_diff": 0.026265908032655716, - "cosine_similarity": 0.9998998045921326, - "kl_divergence": 0.00046225863274036456, - "sigma_level": 119.02215463941639, - "cpk": 39.413532799171506, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.22919845581054688, - "mean_abs_diff": 0.04531732201576233, - "cosine_similarity": 0.9998840689659119, - "kl_divergence": 0.0008305399123272189, - "sigma_level": 72.06274536754663, - "cpk": 23.74877423608546, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.23381578922271729, - "mean_abs_diff": 0.030515603721141815, - "cosine_similarity": 0.9999016523361206, - "kl_divergence": 0.0005923224650756628, - "sigma_level": 103.29603430637736, - "cpk": 34.169333031387545, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.19087600708007812, - "mean_abs_diff": 0.030218342319130898, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.00045970970727517754, - "sigma_level": 104.27458896076459, - "cpk": 34.49561255155461, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2747793197631836, - "mean_abs_diff": 0.03750928118824959, - "cosine_similarity": 0.9999193549156189, - "kl_divergence": 0.00287010815315867, - "sigma_level": 84.41037393510442, - "cpk": 27.872943607440046, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.18088853359222412, - "mean_abs_diff": 0.029429737478494644, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.0002790777073941985, - "sigma_level": 106.92372344549646, - "cpk": 35.37901305589681, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.200392484664917, - "mean_abs_diff": 0.03397694602608681, - "cosine_similarity": 0.9998542070388794, - "kl_divergence": 0.0003240177982787965, - "sigma_level": 94.48294788690083, - "cpk": 31.226795793905374, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.29866647720336914, - "mean_abs_diff": 0.045978154987096786, - "cosine_similarity": 0.9998794198036194, - "kl_divergence": 0.0009558542296503378, - "sigma_level": 68.50529301922718, - "cpk": 22.572618758086186, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.21673345565795898, - "mean_abs_diff": 0.03529226407408714, - "cosine_similarity": 0.9999400973320007, - "kl_divergence": 0.002097586731727194, - "sigma_level": 90.45730879559231, - "cpk": 29.886399329410327, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.22467482089996338, - "mean_abs_diff": 0.03361072018742561, - "cosine_similarity": 0.999942421913147, - "kl_divergence": 0.0008436583035788792, - "sigma_level": 93.8695872897644, - "cpk": 31.026943727212682, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.22781848907470703, - "mean_abs_diff": 0.030904922634363174, - "cosine_similarity": 0.9999483227729797, - "kl_divergence": 0.0008333849906099469, - "sigma_level": 102.17588221636045, - "cpk": 33.795482594203946, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1783003807067871, - "mean_abs_diff": 0.029902499169111252, - "cosine_similarity": 0.9999409317970276, - "kl_divergence": 0.0003094355721386634, - "sigma_level": 106.19563387482063, - "cpk": 35.13391838713145, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2627032399177551, - "mean_abs_diff": 0.03310393542051315, - "cosine_similarity": 0.9999225735664368, - "kl_divergence": 0.00015295545004923058, - "sigma_level": 94.89749613646042, - "cpk": 31.37070866351432, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.19065427780151367, - "mean_abs_diff": 0.028850441798567772, - "cosine_similarity": 0.9999325275421143, - "kl_divergence": 0.00038527130067871274, - "sigma_level": 108.36687717839267, - "cpk": 35.861756369220245, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.23432064056396484, - "mean_abs_diff": 0.037413571029901505, - "cosine_similarity": 0.9999155402183533, - "kl_divergence": 0.0002889698267149911, - "sigma_level": 85.30887203134782, - "cpk": 28.17031488184714, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.18627315759658813, - "mean_abs_diff": 0.02875906601548195, - "cosine_similarity": 0.9999480843544006, - "kl_divergence": 0.0007701578612177225, - "sigma_level": 108.96690810906699, - "cpk": 36.06115382770469, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.22917795181274414, - "mean_abs_diff": 0.03641311824321747, - "cosine_similarity": 0.9999172687530518, - "kl_divergence": 0.0005103069102512929, - "sigma_level": 87.72677177564587, - "cpk": 28.97605681573509, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.24593007564544678, - "mean_abs_diff": 0.03212062641978264, - "cosine_similarity": 0.9998915791511536, - "kl_divergence": 0.00003151487057192769, - "sigma_level": 97.4480464319994, - "cpk": 32.221841119434806, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.19705867767333984, - "mean_abs_diff": 0.0312761552631855, - "cosine_similarity": 0.9999309778213501, - "kl_divergence": 0.0001150733661477417, - "sigma_level": 100.417536631287, - "cpk": 33.210789338193436, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.1876506805419922, - "mean_abs_diff": 0.031015293672680855, - "cosine_similarity": 0.9999198913574219, - "kl_divergence": 0.0007250441446097092, - "sigma_level": 101.70368336368918, - "cpk": 33.638363653969876, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.19601619243621826, - "mean_abs_diff": 0.03426588699221611, - "cosine_similarity": 0.999900221824646, - "kl_divergence": 0.00023789191270330738, - "sigma_level": 93.12063513483277, - "cpk": 30.77430661492979, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.20610380172729492, - "mean_abs_diff": 0.03480095788836479, - "cosine_similarity": 0.9999322891235352, - "kl_divergence": 0.00005894530450833747, - "sigma_level": 92.1492706828521, - "cpk": 30.44918332024425, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.21353816986083984, - "mean_abs_diff": 0.032337937504053116, - "cosine_similarity": 0.9999272227287292, - "kl_divergence": 0.0005776968588551011, - "sigma_level": 96.96144440466122, - "cpk": 32.05918704076535, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.18230438232421875, - "mean_abs_diff": 0.029304036870598793, - "cosine_similarity": 0.9999476671218872, - "kl_divergence": 0.0005154337420336456, - "sigma_level": 107.47869468270946, - "cpk": 35.56376825825433, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.1879286766052246, - "mean_abs_diff": 0.029841218143701553, - "cosine_similarity": 0.9999344348907471, - "kl_divergence": 0.0001249257069489228, - "sigma_level": 104.64479499892016, - "cpk": 34.62137098670958, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.20756947994232178, - "mean_abs_diff": 0.029503241181373596, - "cosine_similarity": 0.9999513030052185, - "kl_divergence": 0.0007071416731914378, - "sigma_level": 106.82336906210507, - "cpk": 35.34515338559784, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.1847095489501953, - "mean_abs_diff": 0.028994187712669373, - "cosine_similarity": 0.9999274611473083, - "kl_divergence": 0.00020641965149885807, - "sigma_level": 107.64212517799307, - "cpk": 35.62062539406424, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.1772775650024414, - "mean_abs_diff": 0.029128575697541237, - "cosine_similarity": 0.9999530911445618, - "kl_divergence": 0.0001424306189745565, - "sigma_level": 107.36934004276407, - "cpk": 35.5291536851688, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.19080734252929688, - "mean_abs_diff": 0.024822678416967392, - "cosine_similarity": 0.9999489188194275, - "kl_divergence": 0.0004685865991592646, - "sigma_level": 125.75471839980924, - "cpk": 41.65810872208185, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.1714634895324707, - "mean_abs_diff": 0.02789604291319847, - "cosine_similarity": 0.9999091029167175, - "kl_divergence": 0.00015832598961693735, - "sigma_level": 112.04260562737294, - "cpk": 37.08707309790033, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.20160150527954102, - "mean_abs_diff": 0.03052336722612381, - "cosine_similarity": 0.9999374747276306, - "kl_divergence": 0.00035990273757939453, - "sigma_level": 103.43257387477989, - "cpk": 34.21443208863304, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.1712021827697754, - "mean_abs_diff": 0.026527106761932373, - "cosine_similarity": 0.9999428391456604, - "kl_divergence": 0.0004203473145415497, - "sigma_level": 117.54782949495923, - "cpk": 38.92275951309922, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.19522953033447266, - "mean_abs_diff": 0.028979429975152016, - "cosine_similarity": 0.9999217987060547, - "kl_divergence": 0.000012096107530428578, - "sigma_level": 108.97087819271881, - "cpk": 36.060466569746545, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.21483612060546875, - "mean_abs_diff": 0.03282041847705841, - "cosine_similarity": 0.999923586845398, - "kl_divergence": 0.0003550886859526683, - "sigma_level": 95.4968733707598, - "cpk": 31.571103844646682, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.19664621353149414, - "mean_abs_diff": 0.03151273727416992, - "cosine_similarity": 0.9999220967292786, - "kl_divergence": 0.0005721978732455467, - "sigma_level": 99.34248182400384, - "cpk": 32.853281147177626, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.23045682907104492, - "mean_abs_diff": 0.048200823366642, - "cosine_similarity": 0.9998908638954163, - "kl_divergence": 0.0011748968653306612, - "sigma_level": 69.28219447916557, - "cpk": 22.815776591509888, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2098684310913086, - "mean_abs_diff": 0.030793752521276474, - "cosine_similarity": 0.9999160766601562, - "kl_divergence": 0.00023351596473067916, - "sigma_level": 102.0201738769548, - "cpk": 33.74492596010622, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.25650787353515625, - "mean_abs_diff": 0.037612028419971466, - "cosine_similarity": 0.9999205470085144, - "kl_divergence": 0.0009010070401926103, - "sigma_level": 85.49162609615232, - "cpk": 28.229249242850933, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.19126582145690918, - "mean_abs_diff": 0.029047662392258644, - "cosine_similarity": 0.9999561905860901, - "kl_divergence": 0.0006576481104213344, - "sigma_level": 107.47869468270946, - "cpk": 35.566064491111156, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.20811530947685242, - "mean_abs_diff": 0.03056376241147518, - "cosine_similarity": 0.9999154210090637, - "kl_divergence": 0.0003487269693072701, - "sigma_level": 102.75772973693056, - "cpk": 33.990854675841675, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.22626805305480957, - "mean_abs_diff": 0.03359080106019974, - "cosine_similarity": 0.9998672008514404, - "kl_divergence": 0.0007366287035674455, - "sigma_level": 93.41415606507609, - "cpk": 30.876563993976326, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.8031983375549316, + "mean_abs_diff": 0.12188125401735306, + "cosine_similarity": 0.9984647035598755, + "kl_divergence": 0.011949143213562554, + "sigma_level": 26.330537211297408, + "cpk": 8.50941249591051, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.24585700035095215, + "mean_abs_diff": 0.03762512281537056, + "cosine_similarity": 0.9998586177825928, + "kl_divergence": 1.3277331549664009e-05, + "sigma_level": 84.0184221427533, + "cpk": 27.742707093261536, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.2099609375, + "mean_abs_diff": 0.033425211906433105, + "cosine_similarity": 0.9999147653579712, + "kl_divergence": 1.5090158264594303e-06, + "sigma_level": 94.57560609882762, + "cpk": 31.261767893356485, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.22704386711120605, + "mean_abs_diff": 0.0315629281103611, + "cosine_similarity": 0.999933660030365, + "kl_divergence": 3.7310540638633336e-05, + "sigma_level": 99.18469092152263, + "cpk": 32.80068370140717, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.2088146209716797, + "mean_abs_diff": 0.03077378123998642, + "cosine_similarity": 0.9999191164970398, + "kl_divergence": 1.4999781410245668e-06, + "sigma_level": 101.59603971600392, + "cpk": 33.60480538024553, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2547646760940552, + "mean_abs_diff": 0.03083600476384163, + "cosine_similarity": 0.9999075531959534, + "kl_divergence": 0.0002142816112921191, + "sigma_level": 101.00436248100775, + "cpk": 33.40857324344982, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.20438094437122345, + "mean_abs_diff": 0.031773317605257034, + "cosine_similarity": 0.9999481439590454, + "kl_divergence": 9.281385733172737e-07, + "sigma_level": 99.44615585236863, + "cpk": 32.88540742624628, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.18281269073486328, + "mean_abs_diff": 0.030860792845487595, + "cosine_similarity": 0.9999558925628662, + "kl_divergence": 2.0817027272510226e-06, + "sigma_level": 102.06207588666598, + "cpk": 33.75821558044542, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.28357791900634766, + "mean_abs_diff": 0.03390377387404442, + "cosine_similarity": 0.999890148639679, + "kl_divergence": 0.00045742262385036316, + "sigma_level": 92.77850864421463, + "cpk": 30.6640410832845, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2932462692260742, + "mean_abs_diff": 0.044488128274679184, + "cosine_similarity": 0.9998958706855774, + "kl_divergence": 0.0016326477262920424, + "sigma_level": 71.4951399913799, + "cpk": 23.566656250547247, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.25107574462890625, + "mean_abs_diff": 0.036512766033411026, + "cosine_similarity": 0.9999284148216248, + "kl_divergence": 0.0011823933043749269, + "sigma_level": 85.6818989695192, + "cpk": 28.299926062308696, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.25175952911376953, + "mean_abs_diff": 0.04251187667250633, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.000520927378713784, + "sigma_level": 75.90532316155377, + "cpk": 25.032867909098698, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.23739862442016602, + "mean_abs_diff": 0.035071682184934616, + "cosine_similarity": 0.9999188184738159, + "kl_divergence": 0.00016192173524096987, + "sigma_level": 91.2163777694204, + "cpk": 30.13886660554096, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.20715034008026123, + "mean_abs_diff": 0.031439900398254395, + "cosine_similarity": 0.9999179244041443, + "kl_divergence": 0.0007082983479816206, + "sigma_level": 100.4901828181864, + "cpk": 33.23344416116162, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.23068474233150482, + "mean_abs_diff": 0.03183136135339737, + "cosine_similarity": 0.9999409914016724, + "kl_divergence": 0.0007547449349859649, + "sigma_level": 98.39571148132515, + "cpk": 32.53756470645949, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23305463790893555, + "mean_abs_diff": 0.028933987021446228, + "cosine_similarity": 0.9998893141746521, + "kl_divergence": 0.0003845604489353084, + "sigma_level": 107.47455288493924, + "cpk": 35.565712018454036, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2608985900878906, + "mean_abs_diff": 0.04424131289124489, + "cosine_similarity": 0.9999105930328369, + "kl_divergence": 0.0007788479711618704, + "sigma_level": 73.24530032745929, + "cpk": 24.14506108835307, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.2301793098449707, + "mean_abs_diff": 0.0324413925409317, + "cosine_similarity": 0.9999143481254578, + "kl_divergence": 2.250089535157956e-05, + "sigma_level": 96.25744820928855, + "cpk": 31.825588931234023, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.19923532009124756, + "mean_abs_diff": 0.03906163200736046, + "cosine_similarity": 0.999877393245697, + "kl_divergence": 0.0005163370630241004, + "sigma_level": 83.7633797367138, + "cpk": 27.648465385990573, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 5942, + "gpu_argmax": 5942, + "max_abs_diff": 0.17966842651367188, + "mean_abs_diff": 0.02848205156624317, + "cosine_similarity": 0.9999198317527771, + "kl_divergence": 0.0008812988738490747, + "sigma_level": 109.6664749254824, + "cpk": 36.29519779233449, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.29142284393310547, + "mean_abs_diff": 0.043598152697086334, + "cosine_similarity": 0.9998776316642761, + "kl_divergence": 0.0010248388343692954, + "sigma_level": 73.4393195777277, + "cpk": 24.21295497016591, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.34327876567840576, + "mean_abs_diff": 0.0488663986325264, + "cosine_similarity": 0.9999039173126221, + "kl_divergence": 0.0018092925711578334, + "sigma_level": 64.72170503736106, + "cpk": 21.310341959242646, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3739614486694336, + "mean_abs_diff": 0.04563745856285095, + "cosine_similarity": 0.9998410940170288, + "kl_divergence": 0.0013028657116986625, + "sigma_level": 68.52047136061793, + "cpk": 22.579565439170377, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.25923848152160645, + "mean_abs_diff": 0.04782666265964508, + "cosine_similarity": 0.9998435378074646, + "kl_divergence": 0.00017440859928140556, + "sigma_level": 68.0917989073902, + "cpk": 22.425882677777388, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.20351624488830566, + "mean_abs_diff": 0.030075596645474434, + "cosine_similarity": 0.9998764991760254, + "kl_divergence": 0.0005101532884886966, + "sigma_level": 103.98703355346954, + "cpk": 34.40172184469705, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.20961999893188477, + "mean_abs_diff": 0.0313001312315464, + "cosine_similarity": 0.9998053312301636, + "kl_divergence": 2.8302109163839864e-05, + "sigma_level": 102.59202935793, + "cpk": 33.929747787458844, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19363737106323242, + "mean_abs_diff": 0.033073790371418, + "cosine_similarity": 0.9998298287391663, + "kl_divergence": 0.0005171189703267121, + "sigma_level": 96.9427542695765, + "cpk": 32.04706272879717, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.20902681350708008, + "mean_abs_diff": 0.03569749742746353, + "cosine_similarity": 0.9997431039810181, + "kl_divergence": 0.0001146219937046406, + "sigma_level": 90.59358669300643, + "cpk": 29.92836520367562, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.20511221885681152, + "mean_abs_diff": 0.03204226866364479, + "cosine_similarity": 0.9998698830604553, + "kl_divergence": 0.0009168103274381441, + "sigma_level": 97.91782700745156, + "cpk": 32.3778165591561, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.20564067363739014, + "mean_abs_diff": 0.03282441571354866, + "cosine_similarity": 0.9998794794082642, + "kl_divergence": 3.538160864634364e-05, + "sigma_level": 96.38664467254054, + "cpk": 31.865228616349718, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2244204878807068, + "mean_abs_diff": 0.048710912466049194, + "cosine_similarity": 0.9996637105941772, + "kl_divergence": 3.898172129763024e-06, + "sigma_level": 69.19204623397793, + "cpk": 22.7831481023718, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19256067276000977, + "mean_abs_diff": 0.038800157606601715, + "cosine_similarity": 0.9998534321784973, + "kl_divergence": 0.00014612402448690114, + "sigma_level": 83.81763111733832, + "cpk": 27.668198930982378, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.24395322799682617, + "mean_abs_diff": 0.035089071840047836, + "cosine_similarity": 0.9998824596405029, + "kl_divergence": 0.0007183325325718559, + "sigma_level": 90.07186375980564, + "cpk": 29.76057641174898, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.22237825393676758, + "mean_abs_diff": 0.030163198709487915, + "cosine_similarity": 0.9998973608016968, + "kl_divergence": 0.0007083169726316857, + "sigma_level": 103.65308835461354, + "cpk": 34.290487059796824, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.19781363010406494, + "mean_abs_diff": 0.028445789590477943, + "cosine_similarity": 0.9998837113380432, + "kl_divergence": 9.986173050612662e-06, + "sigma_level": 109.84427252512704, + "cpk": 36.35437358637827, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.21171927452087402, + "mean_abs_diff": 0.0331043042242527, + "cosine_similarity": 0.999922513961792, + "kl_divergence": 0.0005099449753999734, + "sigma_level": 95.18447747645524, + "cpk": 31.465574500501155, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.2000875473022461, + "mean_abs_diff": 0.028575299307703972, + "cosine_similarity": 0.9998475313186646, + "kl_divergence": 0.001018203258176856, + "sigma_level": 109.12304284617422, + "cpk": 36.11449564783334, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.18700814247131348, + "mean_abs_diff": 0.026265908032655716, + "cosine_similarity": 0.9998998045921326, + "kl_divergence": 0.00046225863274036456, + "sigma_level": 119.02215463941639, + "cpk": 39.413532799171506, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.22919845581054688, + "mean_abs_diff": 0.04531732201576233, + "cosine_similarity": 0.9998840689659119, + "kl_divergence": 0.0008305399123272189, + "sigma_level": 72.06274536754663, + "cpk": 23.74877423608546, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.23381578922271729, + "mean_abs_diff": 0.030515603721141815, + "cosine_similarity": 0.9999016523361206, + "kl_divergence": 0.0005923224650756628, + "sigma_level": 103.29603430637736, + "cpk": 34.169333031387545, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.19087600708007812, + "mean_abs_diff": 0.030218342319130898, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 0.00045970970727517754, + "sigma_level": 104.27458896076459, + "cpk": 34.49561255155461, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2747793197631836, + "mean_abs_diff": 0.03750928118824959, + "cosine_similarity": 0.9999193549156189, + "kl_divergence": 0.00287010815315867, + "sigma_level": 84.41037393510442, + "cpk": 27.872943607440046, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.18088853359222412, + "mean_abs_diff": 0.029429737478494644, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 0.0002790777073941985, + "sigma_level": 106.92372344549646, + "cpk": 35.37901305589681, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.200392484664917, + "mean_abs_diff": 0.03397694602608681, + "cosine_similarity": 0.9998542070388794, + "kl_divergence": 0.0003240177982787965, + "sigma_level": 94.48294788690083, + "cpk": 31.226795793905374, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.29866647720336914, + "mean_abs_diff": 0.045978154987096786, + "cosine_similarity": 0.9998794198036194, + "kl_divergence": 0.0009558542296503378, + "sigma_level": 68.50529301922718, + "cpk": 22.572618758086186, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.21673345565795898, + "mean_abs_diff": 0.03529226407408714, + "cosine_similarity": 0.9999400973320007, + "kl_divergence": 0.002097586731727194, + "sigma_level": 90.45730879559231, + "cpk": 29.886399329410327, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.22467482089996338, + "mean_abs_diff": 0.03361072018742561, + "cosine_similarity": 0.999942421913147, + "kl_divergence": 0.0008436583035788792, + "sigma_level": 93.8695872897644, + "cpk": 31.026943727212682, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.22781848907470703, + "mean_abs_diff": 0.030904922634363174, + "cosine_similarity": 0.9999483227729797, + "kl_divergence": 0.0008333849906099469, + "sigma_level": 102.17588221636045, + "cpk": 33.795482594203946, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1783003807067871, + "mean_abs_diff": 0.029902499169111252, + "cosine_similarity": 0.9999409317970276, + "kl_divergence": 0.0003094355721386634, + "sigma_level": 106.19563387482063, + "cpk": 35.13391838713145, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2627032399177551, + "mean_abs_diff": 0.03310393542051315, + "cosine_similarity": 0.9999225735664368, + "kl_divergence": 0.00015295545004923058, + "sigma_level": 94.89749613646042, + "cpk": 31.37070866351432, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.19065427780151367, + "mean_abs_diff": 0.028850441798567772, + "cosine_similarity": 0.9999325275421143, + "kl_divergence": 0.00038527130067871274, + "sigma_level": 108.36687717839267, + "cpk": 35.861756369220245, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.23432064056396484, + "mean_abs_diff": 0.037413571029901505, + "cosine_similarity": 0.9999155402183533, + "kl_divergence": 0.0002889698267149911, + "sigma_level": 85.30887203134782, + "cpk": 28.17031488184714, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.18627315759658813, + "mean_abs_diff": 0.02875906601548195, + "cosine_similarity": 0.9999480843544006, + "kl_divergence": 0.0007701578612177225, + "sigma_level": 108.96690810906699, + "cpk": 36.06115382770469, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.22917795181274414, + "mean_abs_diff": 0.03641311824321747, + "cosine_similarity": 0.9999172687530518, + "kl_divergence": 0.0005103069102512929, + "sigma_level": 87.72677177564587, + "cpk": 28.97605681573509, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.24593007564544678, + "mean_abs_diff": 0.03212062641978264, + "cosine_similarity": 0.9998915791511536, + "kl_divergence": 3.151487057192769e-05, + "sigma_level": 97.4480464319994, + "cpk": 32.221841119434806, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.19705867767333984, + "mean_abs_diff": 0.0312761552631855, + "cosine_similarity": 0.9999309778213501, + "kl_divergence": 0.0001150733661477417, + "sigma_level": 100.417536631287, + "cpk": 33.210789338193436, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.1876506805419922, + "mean_abs_diff": 0.031015293672680855, + "cosine_similarity": 0.9999198913574219, + "kl_divergence": 0.0007250441446097092, + "sigma_level": 101.70368336368918, + "cpk": 33.638363653969876, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.19601619243621826, + "mean_abs_diff": 0.03426588699221611, + "cosine_similarity": 0.999900221824646, + "kl_divergence": 0.00023789191270330738, + "sigma_level": 93.12063513483277, + "cpk": 30.77430661492979, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.20610380172729492, + "mean_abs_diff": 0.03480095788836479, + "cosine_similarity": 0.9999322891235352, + "kl_divergence": 5.894530450833747e-05, + "sigma_level": 92.1492706828521, + "cpk": 30.44918332024425, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.21353816986083984, + "mean_abs_diff": 0.032337937504053116, + "cosine_similarity": 0.9999272227287292, + "kl_divergence": 0.0005776968588551011, + "sigma_level": 96.96144440466122, + "cpk": 32.05918704076535, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.18230438232421875, + "mean_abs_diff": 0.029304036870598793, + "cosine_similarity": 0.9999476671218872, + "kl_divergence": 0.0005154337420336456, + "sigma_level": 107.47869468270946, + "cpk": 35.56376825825433, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.1879286766052246, + "mean_abs_diff": 0.029841218143701553, + "cosine_similarity": 0.9999344348907471, + "kl_divergence": 0.0001249257069489228, + "sigma_level": 104.64479499892016, + "cpk": 34.62137098670958, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.20756947994232178, + "mean_abs_diff": 0.029503241181373596, + "cosine_similarity": 0.9999513030052185, + "kl_divergence": 0.0007071416731914378, + "sigma_level": 106.82336906210507, + "cpk": 35.34515338559784, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.1847095489501953, + "mean_abs_diff": 0.028994187712669373, + "cosine_similarity": 0.9999274611473083, + "kl_divergence": 0.00020641965149885807, + "sigma_level": 107.64212517799307, + "cpk": 35.62062539406424, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.1772775650024414, + "mean_abs_diff": 0.029128575697541237, + "cosine_similarity": 0.9999530911445618, + "kl_divergence": 0.0001424306189745565, + "sigma_level": 107.36934004276407, + "cpk": 35.5291536851688, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.19080734252929688, + "mean_abs_diff": 0.024822678416967392, + "cosine_similarity": 0.9999489188194275, + "kl_divergence": 0.0004685865991592646, + "sigma_level": 125.75471839980924, + "cpk": 41.65810872208185, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.1714634895324707, + "mean_abs_diff": 0.02789604291319847, + "cosine_similarity": 0.9999091029167175, + "kl_divergence": 0.00015832598961693735, + "sigma_level": 112.04260562737294, + "cpk": 37.08707309790033, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.20160150527954102, + "mean_abs_diff": 0.03052336722612381, + "cosine_similarity": 0.9999374747276306, + "kl_divergence": 0.00035990273757939453, + "sigma_level": 103.43257387477989, + "cpk": 34.21443208863304, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.1712021827697754, + "mean_abs_diff": 0.026527106761932373, + "cosine_similarity": 0.9999428391456604, + "kl_divergence": 0.0004203473145415497, + "sigma_level": 117.54782949495923, + "cpk": 38.92275951309922, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.19522953033447266, + "mean_abs_diff": 0.028979429975152016, + "cosine_similarity": 0.9999217987060547, + "kl_divergence": 1.2096107530428578e-05, + "sigma_level": 108.97087819271881, + "cpk": 36.060466569746545, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.21483612060546875, + "mean_abs_diff": 0.03282041847705841, + "cosine_similarity": 0.999923586845398, + "kl_divergence": 0.0003550886859526683, + "sigma_level": 95.4968733707598, + "cpk": 31.571103844646682, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.19664621353149414, + "mean_abs_diff": 0.03151273727416992, + "cosine_similarity": 0.9999220967292786, + "kl_divergence": 0.0005721978732455467, + "sigma_level": 99.34248182400384, + "cpk": 32.853281147177626, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.23045682907104492, + "mean_abs_diff": 0.048200823366642, + "cosine_similarity": 0.9998908638954163, + "kl_divergence": 0.0011748968653306612, + "sigma_level": 69.28219447916557, + "cpk": 22.815776591509888, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2098684310913086, + "mean_abs_diff": 0.030793752521276474, + "cosine_similarity": 0.9999160766601562, + "kl_divergence": 0.00023351596473067916, + "sigma_level": 102.0201738769548, + "cpk": 33.74492596010622, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.25650787353515625, + "mean_abs_diff": 0.037612028419971466, + "cosine_similarity": 0.9999205470085144, + "kl_divergence": 0.0009010070401926103, + "sigma_level": 85.49162609615232, + "cpk": 28.229249242850933, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.19126582145690918, + "mean_abs_diff": 0.029047662392258644, + "cosine_similarity": 0.9999561905860901, + "kl_divergence": 0.0006576481104213344, + "sigma_level": 107.47869468270946, + "cpk": 35.566064491111156, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.20811530947685242, + "mean_abs_diff": 0.03056376241147518, + "cosine_similarity": 0.9999154210090637, + "kl_divergence": 0.0003487269693072701, + "sigma_level": 102.75772973693056, + "cpk": 33.990854675841675, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.22626805305480957, + "mean_abs_diff": 0.03359080106019974, + "cosine_similarity": 0.9998672008514404, + "kl_divergence": 0.0007366287035674455, + "sigma_level": 93.41415606507609, + "cpk": 30.876563993976326, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json index ce968eac37..20ddb8ff92 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1068 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-1.5b-instruct-q4_k_m", + "file": "~/models/qwen2.5-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M", + "model_sha256": "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.66.0", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "342f4199a612c842", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md \u00a7'2026-09-09 \u2014 the reporter's exact file, measured'", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`.", + "model_sha256_basis": "evidence/parity/l0-1/lambda/RECORD.md \u2014 fetched from Qwen/Qwen2.5-1.5B-Instruct-GGUF, 1117320736 bytes, verified against the HF LFS oid at measurement time" + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9978237152099609, + "min_cosine_position": 4, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "~/models/qwen2.5-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 2701, - "gpu_argmax": 2701, - "max_abs_diff": 0.31842291355133057, - "mean_abs_diff": 0.05168678238987923, - "cosine_similarity": 0.9998032450675964, - "kl_divergence": 0.0010041628555806199, - "sigma_level": 61.438975798323675, - "cpk": 20.215026685079142, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.6011629104614258, - "mean_abs_diff": 0.09295977652072906, - "cosine_similarity": 0.9993610382080078, - "kl_divergence": 0.00038596963427606305, - "sigma_level": 33.920919979632444, - "cpk": 11.044199898153801, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 1.074141263961792, - "mean_abs_diff": 0.1629197895526886, - "cosine_similarity": 0.998677670955658, - "kl_divergence": 0.00015600067604519665, - "sigma_level": 19.209160557495856, - "cpk": 6.142257486206033, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.8302667140960693, - "mean_abs_diff": 0.14872857928276062, - "cosine_similarity": 0.9985498189926147, - "kl_divergence": 0.00035117984368604677, - "sigma_level": 21.646905131259768, - "cpk": 6.947342256583176, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 1.2988052368164062, - "mean_abs_diff": 0.2101205289363861, - "cosine_similarity": 0.9978237152099609, - "kl_divergence": 0.0007265903462690505, - "sigma_level": 15.178418190254549, - "cpk": 4.793697958538687, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 1.0872032642364502, - "mean_abs_diff": 0.16018898785114288, - "cosine_similarity": 0.9980590343475342, - "kl_divergence": 0.013828814951459963, - "sigma_level": 19.748898118047645, - "cpk": 6.3193363726237655, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.9903631210327148, - "mean_abs_diff": 0.17371346056461334, - "cosine_similarity": 0.9984952807426453, - "kl_divergence": 0.0038416856287773597, - "sigma_level": 18.438460185295785, - "cpk": 5.879236001242714, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.7914443016052246, - "mean_abs_diff": 0.13904337584972382, - "cosine_similarity": 0.9994173645973206, - "kl_divergence": 0.0011267766165267705, - "sigma_level": 23.019298841074793, - "cpk": 7.406376195311875, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 624, - "gpu_argmax": 624, - "max_abs_diff": 0.7683401107788086, - "mean_abs_diff": 0.11628174781799316, - "cosine_similarity": 0.9988746643066406, - "kl_divergence": 0.0021637816443614774, - "sigma_level": 27.418492188242922, - "cpk": 8.873808213232401, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.5380454063415527, - "mean_abs_diff": 0.10169024765491486, - "cosine_similarity": 0.999545693397522, - "kl_divergence": 0.004308393425260426, - "sigma_level": 31.753008418995396, - "cpk": 10.315255198838779, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 3974, - "gpu_argmax": 3974, - "max_abs_diff": 0.4585554599761963, - "mean_abs_diff": 0.06840988248586655, - "cosine_similarity": 0.9997379183769226, - "kl_divergence": 0.003921000934134561, - "sigma_level": 46.307180649881325, - "cpk": 15.17173781775126, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.5177440643310547, - "mean_abs_diff": 0.06941164284944534, - "cosine_similarity": 0.9996582865715027, - "kl_divergence": 0.0020656723212676966, - "sigma_level": 45.76503932690633, - "cpk": 14.99029422857294, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.420398473739624, - "mean_abs_diff": 0.06813345104455948, - "cosine_similarity": 0.9997531175613403, - "kl_divergence": 0.0012232872752691486, - "sigma_level": 46.26818246767472, - "cpk": 15.160026577134735, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.4519714117050171, - "mean_abs_diff": 0.0760267823934555, - "cosine_similarity": 0.9996703267097473, - "kl_divergence": 0.001647045695020044, - "sigma_level": 41.99132346083567, - "cpk": 13.731069052681043, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 9293, - "gpu_argmax": 9293, - "max_abs_diff": 0.44501709938049316, - "mean_abs_diff": 0.08212687820196152, - "cosine_similarity": 0.999630331993103, - "kl_divergence": 0.006501142529339967, - "sigma_level": 38.93182174829522, - "cpk": 12.710828167523179, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 369, - "gpu_argmax": 369, - "max_abs_diff": 0.46442174911499023, - "mean_abs_diff": 0.08806511759757996, - "cosine_similarity": 0.9996541738510132, - "kl_divergence": 0.0013393907643138331, - "sigma_level": 36.201662750395194, - "cpk": 11.801545609519941, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.41184115409851074, - "mean_abs_diff": 0.0735999047756195, - "cosine_similarity": 0.9997572898864746, - "kl_divergence": 0.0020352302058443735, - "sigma_level": 42.657347277985636, - "cpk": 13.957484367858523, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.39466023445129395, - "mean_abs_diff": 0.07817857712507248, - "cosine_similarity": 0.999573826789856, - "kl_divergence": 0.00016666892717610718, - "sigma_level": 41.49852594567503, - "cpk": 13.562484005956613, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.5057002902030945, - "mean_abs_diff": 0.07810698449611664, - "cosine_similarity": 0.9997146129608154, - "kl_divergence": 0.0018880510517696614, - "sigma_level": 40.865303812203365, - "cpk": 13.355779133127049, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 14310, - "gpu_argmax": 14310, - "max_abs_diff": 0.4656808376312256, - "mean_abs_diff": 0.07576054334640503, - "cosine_similarity": 0.9997501373291016, - "kl_divergence": 0.0019064405359464263, - "sigma_level": 42.13443236727582, - "cpk": 13.778800164947176, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.3974335193634033, - "mean_abs_diff": 0.07983855158090591, - "cosine_similarity": 0.999765157699585, - "kl_divergence": 0.003539420801547969, - "sigma_level": 40.48393264839967, - "cpk": 13.225296004054293, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.5480650663375854, - "mean_abs_diff": 0.09393318742513657, - "cosine_similarity": 0.999376118183136, - "kl_divergence": 0.0038084510052187473, - "sigma_level": 33.93247368937331, - "cpk": 11.045209112219235, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.5794413089752197, - "mean_abs_diff": 0.08977903425693512, - "cosine_similarity": 0.9995113015174866, - "kl_divergence": 0.002239943144359716, - "sigma_level": 34.99617146302287, - "cpk": 11.40356361462093, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.4805694818496704, - "mean_abs_diff": 0.07361047714948654, - "cosine_similarity": 0.9997532367706299, - "kl_divergence": 0.0017476027456557173, - "sigma_level": 43.46233730592578, - "cpk": 14.220838819715166, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.575049638748169, - "mean_abs_diff": 0.08266887068748474, - "cosine_similarity": 0.9996919631958008, - "kl_divergence": 0.005820226039888798, - "sigma_level": 38.466205868760724, - "cpk": 12.55707213968668, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4894747734069824, - "mean_abs_diff": 0.07466553151607513, - "cosine_similarity": 0.9998354911804199, - "kl_divergence": 0.001684058784819267, - "sigma_level": 42.247815077849495, - "cpk": 13.819733728601458, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.39302289485931396, - "mean_abs_diff": 0.0665874183177948, - "cosine_similarity": 0.9998313784599304, - "kl_divergence": 0.0034372790646476988, - "sigma_level": 47.353553428886435, - "cpk": 15.521755237045204, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.41148829460144043, - "mean_abs_diff": 0.0705053061246872, - "cosine_similarity": 0.9995219707489014, - "kl_divergence": 0.000481547916431422, - "sigma_level": 44.680539884015715, - "cpk": 14.63099536614367, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.4649146795272827, - "mean_abs_diff": 0.07782384008169174, - "cosine_similarity": 0.9995982646942139, - "kl_divergence": 0.0015583181127582582, - "sigma_level": 40.227398043234544, - "cpk": 13.148245131726577, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.47322654724121094, - "mean_abs_diff": 0.087704598903656, - "cosine_similarity": 0.9995580315589905, - "kl_divergence": 0.0022127799633092564, - "sigma_level": 36.708986647160415, - "cpk": 11.968033303216066, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.430513858795166, - "mean_abs_diff": 0.08133339881896973, - "cosine_similarity": 0.9997889995574951, - "kl_divergence": 0.00018704778171310847, - "sigma_level": 39.01572660971666, - "cpk": 12.740802065525553, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.45681631565093994, - "mean_abs_diff": 0.08105611801147461, - "cosine_similarity": 0.9997667074203491, - "kl_divergence": 0.003126264705790457, - "sigma_level": 39.84984885704753, - "cpk": 13.01411011470782, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.36823856830596924, - "mean_abs_diff": 0.061732012778520584, - "cosine_similarity": 0.9998225569725037, - "kl_divergence": 0.0028589001523262048, - "sigma_level": 50.88439404151175, - "cpk": 16.699698341904092, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 1075, - "gpu_argmax": 1075, - "max_abs_diff": 0.45197248458862305, - "mean_abs_diff": 0.06759438663721085, - "cosine_similarity": 0.9998044967651367, - "kl_divergence": 0.0044328146591125035, - "sigma_level": 46.359707613196264, - "cpk": 15.192097870999218, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.4539768099784851, - "mean_abs_diff": 0.07912351936101913, - "cosine_similarity": 0.9997790455818176, - "kl_divergence": 0.00013784763067440514, - "sigma_level": 40.56873603366942, - "cpk": 13.255416913638795, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.38960933685302734, - "mean_abs_diff": 0.06396492570638657, - "cosine_similarity": 0.9998103380203247, - "kl_divergence": 0.0029847960362235373, - "sigma_level": 49.1493031479779, - "cpk": 16.121115088960877, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.40729308128356934, - "mean_abs_diff": 0.06924924999475479, - "cosine_similarity": 0.9996633529663086, - "kl_divergence": 0.005449064420257557, - "sigma_level": 45.52429008065333, - "cpk": 14.912053114832036, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.36036384105682373, - "mean_abs_diff": 0.05879916250705719, - "cosine_similarity": 0.9997984170913696, - "kl_divergence": 0.00044588987994617586, - "sigma_level": 53.598492891250764, - "cpk": 17.60353542261309, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 8003, - "gpu_argmax": 17944, - "max_abs_diff": 0.4275398254394531, - "mean_abs_diff": 0.06924005597829819, - "cosine_similarity": 0.9997360110282898, - "kl_divergence": 0.0030625878432387578, - "sigma_level": 45.48962434668141, - "cpk": 14.900732770877472, - "verdict": "WarnArgmax" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 16293, - "gpu_argmax": 16293, - "max_abs_diff": 0.4567922353744507, - "mean_abs_diff": 0.07736830413341522, - "cosine_similarity": 0.9996188879013062, - "kl_divergence": 0.00627417830634242, - "sigma_level": 40.883418450180905, - "cpk": 13.36421608733803, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.42419886589050293, - "mean_abs_diff": 0.07781759649515152, - "cosine_similarity": 0.9997288584709167, - "kl_divergence": 0.00445967416527253, - "sigma_level": 40.85204776562423, - "cpk": 13.352431907789242, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2849, - "gpu_argmax": 2849, - "max_abs_diff": 0.41576290130615234, - "mean_abs_diff": 0.06628850847482681, - "cosine_similarity": 0.9997788071632385, - "kl_divergence": 0.002364125347954264, - "sigma_level": 47.590142022032, - "cpk": 15.600490712948526, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 15409, - "gpu_argmax": 15409, - "max_abs_diff": 0.4992527961730957, - "mean_abs_diff": 0.08212177455425262, - "cosine_similarity": 0.9995926022529602, - "kl_divergence": 0.0024074922898179433, - "sigma_level": 39.21766934638157, - "cpk": 12.804171065409959, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.5810892581939697, - "mean_abs_diff": 0.0682300254702568, - "cosine_similarity": 0.999836266040802, - "kl_divergence": 0.002712155435780897, - "sigma_level": 46.05452141964385, - "cpk": 15.08964870909105, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.43909335136413574, - "mean_abs_diff": 0.07037733495235443, - "cosine_similarity": 0.9998226165771484, - "kl_divergence": 0.0024886335454403913, - "sigma_level": 44.62151623300525, - "cpk": 14.612143461500741, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.5455688238143921, - "mean_abs_diff": 0.08337622880935669, - "cosine_similarity": 0.9997198581695557, - "kl_divergence": 0.004219288243595902, - "sigma_level": 38.8716398899516, - "cpk": 12.68713240151224, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.4702770709991455, - "mean_abs_diff": 0.07033935934305191, - "cosine_similarity": 0.9997967481613159, - "kl_divergence": 0.002269168837427208, - "sigma_level": 44.76284014389586, - "cpk": 14.658564256457195, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.47287213802337646, - "mean_abs_diff": 0.08464169502258301, - "cosine_similarity": 0.9996572136878967, - "kl_divergence": 0.0032647584277852933, - "sigma_level": 38.09939719641655, - "cpk": 12.431065935635237, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.403623104095459, - "mean_abs_diff": 0.08138824999332428, - "cosine_similarity": 0.9997397065162659, - "kl_divergence": 0.002429769070231016, - "sigma_level": 39.845518962009926, - "cpk": 13.011593232470492, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.41884738206863403, - "mean_abs_diff": 0.07357224076986313, - "cosine_similarity": 0.9997681975364685, - "kl_divergence": 0.0026198766285110034, - "sigma_level": 43.19010015433112, - "cpk": 14.131900680824627, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.38974857330322266, - "mean_abs_diff": 0.06174059584736824, - "cosine_similarity": 0.9997618794441223, - "kl_divergence": 0.0024988794185040867, - "sigma_level": 51.257036123035775, - "cpk": 16.8219587117114, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.4303196668624878, - "mean_abs_diff": 0.07956542819738388, - "cosine_similarity": 0.9997139573097229, - "kl_divergence": 0.000753836734309782, - "sigma_level": 40.610606016595504, - "cpk": 13.267601984109694, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.39939403533935547, - "mean_abs_diff": 0.06406113505363464, - "cosine_similarity": 0.9996994137763977, - "kl_divergence": 0.0021454389459888124, - "sigma_level": 49.09465897252818, - "cpk": 16.102798025938455, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.4145479202270508, - "mean_abs_diff": 0.06807403266429901, - "cosine_similarity": 0.9997435808181763, - "kl_divergence": 0.0013922520524832468, - "sigma_level": 46.44678116489198, - "cpk": 15.218775413449793, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.39621949195861816, - "mean_abs_diff": 0.06436076015233994, - "cosine_similarity": 0.9998461008071899, - "kl_divergence": 0.0007129833951081995, - "sigma_level": 48.813617709656256, - "cpk": 16.009399108087152, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.5128415822982788, - "mean_abs_diff": 0.07186087220907211, - "cosine_similarity": 0.9997901320457458, - "kl_divergence": 0.0010620789511583434, - "sigma_level": 44.16569226501575, - "cpk": 14.457415324348458, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.4191385507583618, - "mean_abs_diff": 0.0679415762424469, - "cosine_similarity": 0.9998086094856262, - "kl_divergence": 0.0027788771123882566, - "sigma_level": 46.44338193961324, - "cpk": 15.218174265287137, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.36243438720703125, - "mean_abs_diff": 0.06031492352485657, - "cosine_similarity": 0.9998459815979004, - "kl_divergence": 0.0011979330012718125, - "sigma_level": 52.57512210986416, - "cpk": 17.26078533084085, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.4538910984992981, - "mean_abs_diff": 0.06918986886739731, - "cosine_similarity": 0.9997843503952026, - "kl_divergence": 0.0007243342259276751, - "sigma_level": 45.90852515083764, - "cpk": 15.038141314022376, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.44234466552734375, - "mean_abs_diff": 0.0634784922003746, - "cosine_similarity": 0.9998288154602051, - "kl_divergence": 0.0007674909766671851, - "sigma_level": 49.43904155443516, - "cpk": 16.21815420033612, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.351959228515625, - "mean_abs_diff": 0.05642838776111603, - "cosine_similarity": 0.999832808971405, - "kl_divergence": 0.0019997153002613145, - "sigma_level": 55.90082537578916, - "cpk": 18.370742337723765, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.38826751708984375, - "mean_abs_diff": 0.0642109364271164, - "cosine_similarity": 0.9998274445533752, - "kl_divergence": 0.0006867424058280134, - "sigma_level": 49.15437011428163, - "cpk": 16.121769360216955, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.37309741973876953, - "mean_abs_diff": 0.06227777898311615, - "cosine_similarity": 0.9998220205307007, - "kl_divergence": 0.0021385287649098024, - "sigma_level": 50.534553708367916, - "cpk": 16.58258625555096, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 18589, - "gpu_argmax": 18589, - "max_abs_diff": 0.3214547634124756, - "mean_abs_diff": 0.053083546459674835, - "cosine_similarity": 0.9998363256454468, - "kl_divergence": 0.000752778125991802, - "sigma_level": 59.39349988162695, - "cpk": 19.535098493011564, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.36084842681884766, - "mean_abs_diff": 0.061356306076049805, - "cosine_similarity": 0.9998142123222351, - "kl_divergence": 0.003312728169157689, - "sigma_level": 51.05759445120621, - "cpk": 16.75813936768082, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 7354, - "gpu_argmax": 7354, - "max_abs_diff": 0.44296157360076904, - "mean_abs_diff": 0.06914495676755905, - "cosine_similarity": 0.9997162818908691, - "kl_divergence": 0.0032034377972692007, - "sigma_level": 46.07774351870567, - "cpk": 15.093744207606258, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.4447762966156006, - "mean_abs_diff": 0.07214333862066269, - "cosine_similarity": 0.999789834022522, - "kl_divergence": 0.002128935912980073, - "sigma_level": 44.064428985081875, - "cpk": 14.42323007657755, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1033, - "gpu_argmax": 1033, - "max_abs_diff": 0.43563520908355713, - "mean_abs_diff": 0.07464952021837234, - "cosine_similarity": 0.9997342228889465, - "kl_divergence": 0.0023521526351508127, - "sigma_level": 42.89405481276757, - "cpk": 14.031183219923053, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.41919422149658203, - "mean_abs_diff": 0.07205758988857269, - "cosine_similarity": 0.9996944665908813, - "kl_divergence": 0.0015906886315562518, - "sigma_level": 44.11487020371673, - "cpk": 14.44005579914499, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.4340832233428955, - "mean_abs_diff": 0.06832505017518997, - "cosine_similarity": 0.9997145533561707, - "kl_divergence": 4.531155001990606e-05, - "sigma_level": 46.65343302362604, - "cpk": 15.285511161860004, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.5156793594360352, - "mean_abs_diff": 0.07876264303922653, - "cosine_similarity": 0.9997888803482056, - "kl_divergence": 0.002400534868525818, - "sigma_level": 40.82280172433786, - "cpk": 13.339657928106357, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.49131178855895996, - "mean_abs_diff": 0.06945641338825226, - "cosine_similarity": 0.9997475743293762, - "kl_divergence": 0.0013757893037735654, - "sigma_level": 45.35377881058699, - "cpk": 14.855417036046704, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 2390, - "gpu_argmax": 2390, - "max_abs_diff": 0.5032455325126648, - "mean_abs_diff": 0.07041440159082413, - "cosine_similarity": 0.9997410178184509, - "kl_divergence": 0.0025526453726194957, - "sigma_level": 44.967915760286665, - "cpk": 14.72543951350829, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.47355103492736816, - "mean_abs_diff": 0.08038556575775146, - "cosine_similarity": 0.999701976776123, - "kl_divergence": 0.0016015019282984591, - "sigma_level": 39.61497617254028, - "cpk": 12.939619368170968, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.4618046283721924, - "mean_abs_diff": 0.07349393516778946, - "cosine_similarity": 0.999594509601593, - "kl_divergence": 0.0007789350432452571, - "sigma_level": 43.43102772649833, - "cpk": 14.211016147499297, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 1865, - "max_abs_diff": 0.40026283264160156, - "mean_abs_diff": 0.06240736320614815, - "cosine_similarity": 0.9997082948684692, - "kl_divergence": 0.002672701304671866, - "sigma_level": 50.514450324264075, - "cpk": 16.57544397070925, - "verdict": "WarnArgmax" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.38044023513793945, - "mean_abs_diff": 0.06645609438419342, - "cosine_similarity": 0.9998005628585815, - "kl_divergence": 0.000939163318030381, - "sigma_level": 47.37136548013599, - "cpk": 15.528112165423995, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.4195518493652344, - "mean_abs_diff": 0.06238096207380295, - "cosine_similarity": 0.9997071623802185, - "kl_divergence": 0.0019081753538363014, - "sigma_level": 50.68498889665763, - "cpk": 16.631514768046415, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 2701, + "gpu_argmax": 2701, + "max_abs_diff": 0.31842291355133057, + "mean_abs_diff": 0.05168678238987923, + "cosine_similarity": 0.9998032450675964, + "kl_divergence": 0.0010041628555806199, + "sigma_level": 61.438975798323675, + "cpk": 20.215026685079142, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.6011629104614258, + "mean_abs_diff": 0.09295977652072906, + "cosine_similarity": 0.9993610382080078, + "kl_divergence": 0.00038596963427606305, + "sigma_level": 33.920919979632444, + "cpk": 11.044199898153801, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 1.074141263961792, + "mean_abs_diff": 0.1629197895526886, + "cosine_similarity": 0.998677670955658, + "kl_divergence": 0.00015600067604519665, + "sigma_level": 19.209160557495856, + "cpk": 6.142257486206033, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.8302667140960693, + "mean_abs_diff": 0.14872857928276062, + "cosine_similarity": 0.9985498189926147, + "kl_divergence": 0.00035117984368604677, + "sigma_level": 21.646905131259768, + "cpk": 6.947342256583176, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 1.2988052368164062, + "mean_abs_diff": 0.2101205289363861, + "cosine_similarity": 0.9978237152099609, + "kl_divergence": 0.0007265903462690505, + "sigma_level": 15.178418190254549, + "cpk": 4.793697958538687, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 1.0872032642364502, + "mean_abs_diff": 0.16018898785114288, + "cosine_similarity": 0.9980590343475342, + "kl_divergence": 0.013828814951459963, + "sigma_level": 19.748898118047645, + "cpk": 6.3193363726237655, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.9903631210327148, + "mean_abs_diff": 0.17371346056461334, + "cosine_similarity": 0.9984952807426453, + "kl_divergence": 0.0038416856287773597, + "sigma_level": 18.438460185295785, + "cpk": 5.879236001242714, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.7914443016052246, + "mean_abs_diff": 0.13904337584972382, + "cosine_similarity": 0.9994173645973206, + "kl_divergence": 0.0011267766165267705, + "sigma_level": 23.019298841074793, + "cpk": 7.406376195311875, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 624, + "gpu_argmax": 624, + "max_abs_diff": 0.7683401107788086, + "mean_abs_diff": 0.11628174781799316, + "cosine_similarity": 0.9988746643066406, + "kl_divergence": 0.0021637816443614774, + "sigma_level": 27.418492188242922, + "cpk": 8.873808213232401, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.5380454063415527, + "mean_abs_diff": 0.10169024765491486, + "cosine_similarity": 0.999545693397522, + "kl_divergence": 0.004308393425260426, + "sigma_level": 31.753008418995396, + "cpk": 10.315255198838779, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 3974, + "gpu_argmax": 3974, + "max_abs_diff": 0.4585554599761963, + "mean_abs_diff": 0.06840988248586655, + "cosine_similarity": 0.9997379183769226, + "kl_divergence": 0.003921000934134561, + "sigma_level": 46.307180649881325, + "cpk": 15.17173781775126, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.5177440643310547, + "mean_abs_diff": 0.06941164284944534, + "cosine_similarity": 0.9996582865715027, + "kl_divergence": 0.0020656723212676966, + "sigma_level": 45.76503932690633, + "cpk": 14.99029422857294, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.420398473739624, + "mean_abs_diff": 0.06813345104455948, + "cosine_similarity": 0.9997531175613403, + "kl_divergence": 0.0012232872752691486, + "sigma_level": 46.26818246767472, + "cpk": 15.160026577134735, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.4519714117050171, + "mean_abs_diff": 0.0760267823934555, + "cosine_similarity": 0.9996703267097473, + "kl_divergence": 0.001647045695020044, + "sigma_level": 41.99132346083567, + "cpk": 13.731069052681043, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 9293, + "gpu_argmax": 9293, + "max_abs_diff": 0.44501709938049316, + "mean_abs_diff": 0.08212687820196152, + "cosine_similarity": 0.999630331993103, + "kl_divergence": 0.006501142529339967, + "sigma_level": 38.93182174829522, + "cpk": 12.710828167523179, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 369, + "gpu_argmax": 369, + "max_abs_diff": 0.46442174911499023, + "mean_abs_diff": 0.08806511759757996, + "cosine_similarity": 0.9996541738510132, + "kl_divergence": 0.0013393907643138331, + "sigma_level": 36.201662750395194, + "cpk": 11.801545609519941, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.41184115409851074, + "mean_abs_diff": 0.0735999047756195, + "cosine_similarity": 0.9997572898864746, + "kl_divergence": 0.0020352302058443735, + "sigma_level": 42.657347277985636, + "cpk": 13.957484367858523, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.39466023445129395, + "mean_abs_diff": 0.07817857712507248, + "cosine_similarity": 0.999573826789856, + "kl_divergence": 0.00016666892717610718, + "sigma_level": 41.49852594567503, + "cpk": 13.562484005956613, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.5057002902030945, + "mean_abs_diff": 0.07810698449611664, + "cosine_similarity": 0.9997146129608154, + "kl_divergence": 0.0018880510517696614, + "sigma_level": 40.865303812203365, + "cpk": 13.355779133127049, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 14310, + "gpu_argmax": 14310, + "max_abs_diff": 0.4656808376312256, + "mean_abs_diff": 0.07576054334640503, + "cosine_similarity": 0.9997501373291016, + "kl_divergence": 0.0019064405359464263, + "sigma_level": 42.13443236727582, + "cpk": 13.778800164947176, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.3974335193634033, + "mean_abs_diff": 0.07983855158090591, + "cosine_similarity": 0.999765157699585, + "kl_divergence": 0.003539420801547969, + "sigma_level": 40.48393264839967, + "cpk": 13.225296004054293, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.5480650663375854, + "mean_abs_diff": 0.09393318742513657, + "cosine_similarity": 0.999376118183136, + "kl_divergence": 0.0038084510052187473, + "sigma_level": 33.93247368937331, + "cpk": 11.045209112219235, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.5794413089752197, + "mean_abs_diff": 0.08977903425693512, + "cosine_similarity": 0.9995113015174866, + "kl_divergence": 0.002239943144359716, + "sigma_level": 34.99617146302287, + "cpk": 11.40356361462093, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.4805694818496704, + "mean_abs_diff": 0.07361047714948654, + "cosine_similarity": 0.9997532367706299, + "kl_divergence": 0.0017476027456557173, + "sigma_level": 43.46233730592578, + "cpk": 14.220838819715166, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.575049638748169, + "mean_abs_diff": 0.08266887068748474, + "cosine_similarity": 0.9996919631958008, + "kl_divergence": 0.005820226039888798, + "sigma_level": 38.466205868760724, + "cpk": 12.55707213968668, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4894747734069824, + "mean_abs_diff": 0.07466553151607513, + "cosine_similarity": 0.9998354911804199, + "kl_divergence": 0.001684058784819267, + "sigma_level": 42.247815077849495, + "cpk": 13.819733728601458, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.39302289485931396, + "mean_abs_diff": 0.0665874183177948, + "cosine_similarity": 0.9998313784599304, + "kl_divergence": 0.0034372790646476988, + "sigma_level": 47.353553428886435, + "cpk": 15.521755237045204, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.41148829460144043, + "mean_abs_diff": 0.0705053061246872, + "cosine_similarity": 0.9995219707489014, + "kl_divergence": 0.000481547916431422, + "sigma_level": 44.680539884015715, + "cpk": 14.63099536614367, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.4649146795272827, + "mean_abs_diff": 0.07782384008169174, + "cosine_similarity": 0.9995982646942139, + "kl_divergence": 0.0015583181127582582, + "sigma_level": 40.227398043234544, + "cpk": 13.148245131726577, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.47322654724121094, + "mean_abs_diff": 0.087704598903656, + "cosine_similarity": 0.9995580315589905, + "kl_divergence": 0.0022127799633092564, + "sigma_level": 36.708986647160415, + "cpk": 11.968033303216066, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.430513858795166, + "mean_abs_diff": 0.08133339881896973, + "cosine_similarity": 0.9997889995574951, + "kl_divergence": 0.00018704778171310847, + "sigma_level": 39.01572660971666, + "cpk": 12.740802065525553, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.45681631565093994, + "mean_abs_diff": 0.08105611801147461, + "cosine_similarity": 0.9997667074203491, + "kl_divergence": 0.003126264705790457, + "sigma_level": 39.84984885704753, + "cpk": 13.01411011470782, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.36823856830596924, + "mean_abs_diff": 0.061732012778520584, + "cosine_similarity": 0.9998225569725037, + "kl_divergence": 0.0028589001523262048, + "sigma_level": 50.88439404151175, + "cpk": 16.699698341904092, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 1075, + "gpu_argmax": 1075, + "max_abs_diff": 0.45197248458862305, + "mean_abs_diff": 0.06759438663721085, + "cosine_similarity": 0.9998044967651367, + "kl_divergence": 0.0044328146591125035, + "sigma_level": 46.359707613196264, + "cpk": 15.192097870999218, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.4539768099784851, + "mean_abs_diff": 0.07912351936101913, + "cosine_similarity": 0.9997790455818176, + "kl_divergence": 0.00013784763067440514, + "sigma_level": 40.56873603366942, + "cpk": 13.255416913638795, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.38960933685302734, + "mean_abs_diff": 0.06396492570638657, + "cosine_similarity": 0.9998103380203247, + "kl_divergence": 0.0029847960362235373, + "sigma_level": 49.1493031479779, + "cpk": 16.121115088960877, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.40729308128356934, + "mean_abs_diff": 0.06924924999475479, + "cosine_similarity": 0.9996633529663086, + "kl_divergence": 0.005449064420257557, + "sigma_level": 45.52429008065333, + "cpk": 14.912053114832036, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.36036384105682373, + "mean_abs_diff": 0.05879916250705719, + "cosine_similarity": 0.9997984170913696, + "kl_divergence": 0.00044588987994617586, + "sigma_level": 53.598492891250764, + "cpk": 17.60353542261309, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 8003, + "gpu_argmax": 17944, + "max_abs_diff": 0.4275398254394531, + "mean_abs_diff": 0.06924005597829819, + "cosine_similarity": 0.9997360110282898, + "kl_divergence": 0.0030625878432387578, + "sigma_level": 45.48962434668141, + "cpk": 14.900732770877472, + "verdict": "WarnArgmax" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 16293, + "gpu_argmax": 16293, + "max_abs_diff": 0.4567922353744507, + "mean_abs_diff": 0.07736830413341522, + "cosine_similarity": 0.9996188879013062, + "kl_divergence": 0.00627417830634242, + "sigma_level": 40.883418450180905, + "cpk": 13.36421608733803, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.42419886589050293, + "mean_abs_diff": 0.07781759649515152, + "cosine_similarity": 0.9997288584709167, + "kl_divergence": 0.00445967416527253, + "sigma_level": 40.85204776562423, + "cpk": 13.352431907789242, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2849, + "gpu_argmax": 2849, + "max_abs_diff": 0.41576290130615234, + "mean_abs_diff": 0.06628850847482681, + "cosine_similarity": 0.9997788071632385, + "kl_divergence": 0.002364125347954264, + "sigma_level": 47.590142022032, + "cpk": 15.600490712948526, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 15409, + "gpu_argmax": 15409, + "max_abs_diff": 0.4992527961730957, + "mean_abs_diff": 0.08212177455425262, + "cosine_similarity": 0.9995926022529602, + "kl_divergence": 0.0024074922898179433, + "sigma_level": 39.21766934638157, + "cpk": 12.804171065409959, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.5810892581939697, + "mean_abs_diff": 0.0682300254702568, + "cosine_similarity": 0.999836266040802, + "kl_divergence": 0.002712155435780897, + "sigma_level": 46.05452141964385, + "cpk": 15.08964870909105, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.43909335136413574, + "mean_abs_diff": 0.07037733495235443, + "cosine_similarity": 0.9998226165771484, + "kl_divergence": 0.0024886335454403913, + "sigma_level": 44.62151623300525, + "cpk": 14.612143461500741, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.5455688238143921, + "mean_abs_diff": 0.08337622880935669, + "cosine_similarity": 0.9997198581695557, + "kl_divergence": 0.004219288243595902, + "sigma_level": 38.8716398899516, + "cpk": 12.68713240151224, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.4702770709991455, + "mean_abs_diff": 0.07033935934305191, + "cosine_similarity": 0.9997967481613159, + "kl_divergence": 0.002269168837427208, + "sigma_level": 44.76284014389586, + "cpk": 14.658564256457195, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.47287213802337646, + "mean_abs_diff": 0.08464169502258301, + "cosine_similarity": 0.9996572136878967, + "kl_divergence": 0.0032647584277852933, + "sigma_level": 38.09939719641655, + "cpk": 12.431065935635237, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.403623104095459, + "mean_abs_diff": 0.08138824999332428, + "cosine_similarity": 0.9997397065162659, + "kl_divergence": 0.002429769070231016, + "sigma_level": 39.845518962009926, + "cpk": 13.011593232470492, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.41884738206863403, + "mean_abs_diff": 0.07357224076986313, + "cosine_similarity": 0.9997681975364685, + "kl_divergence": 0.0026198766285110034, + "sigma_level": 43.19010015433112, + "cpk": 14.131900680824627, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.38974857330322266, + "mean_abs_diff": 0.06174059584736824, + "cosine_similarity": 0.9997618794441223, + "kl_divergence": 0.0024988794185040867, + "sigma_level": 51.257036123035775, + "cpk": 16.8219587117114, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.4303196668624878, + "mean_abs_diff": 0.07956542819738388, + "cosine_similarity": 0.9997139573097229, + "kl_divergence": 0.000753836734309782, + "sigma_level": 40.610606016595504, + "cpk": 13.267601984109694, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.39939403533935547, + "mean_abs_diff": 0.06406113505363464, + "cosine_similarity": 0.9996994137763977, + "kl_divergence": 0.0021454389459888124, + "sigma_level": 49.09465897252818, + "cpk": 16.102798025938455, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.4145479202270508, + "mean_abs_diff": 0.06807403266429901, + "cosine_similarity": 0.9997435808181763, + "kl_divergence": 0.0013922520524832468, + "sigma_level": 46.44678116489198, + "cpk": 15.218775413449793, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.39621949195861816, + "mean_abs_diff": 0.06436076015233994, + "cosine_similarity": 0.9998461008071899, + "kl_divergence": 0.0007129833951081995, + "sigma_level": 48.813617709656256, + "cpk": 16.009399108087152, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.5128415822982788, + "mean_abs_diff": 0.07186087220907211, + "cosine_similarity": 0.9997901320457458, + "kl_divergence": 0.0010620789511583434, + "sigma_level": 44.16569226501575, + "cpk": 14.457415324348458, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.4191385507583618, + "mean_abs_diff": 0.0679415762424469, + "cosine_similarity": 0.9998086094856262, + "kl_divergence": 0.0027788771123882566, + "sigma_level": 46.44338193961324, + "cpk": 15.218174265287137, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.36243438720703125, + "mean_abs_diff": 0.06031492352485657, + "cosine_similarity": 0.9998459815979004, + "kl_divergence": 0.0011979330012718125, + "sigma_level": 52.57512210986416, + "cpk": 17.26078533084085, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.4538910984992981, + "mean_abs_diff": 0.06918986886739731, + "cosine_similarity": 0.9997843503952026, + "kl_divergence": 0.0007243342259276751, + "sigma_level": 45.90852515083764, + "cpk": 15.038141314022376, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.44234466552734375, + "mean_abs_diff": 0.0634784922003746, + "cosine_similarity": 0.9998288154602051, + "kl_divergence": 0.0007674909766671851, + "sigma_level": 49.43904155443516, + "cpk": 16.21815420033612, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.351959228515625, + "mean_abs_diff": 0.05642838776111603, + "cosine_similarity": 0.999832808971405, + "kl_divergence": 0.0019997153002613145, + "sigma_level": 55.90082537578916, + "cpk": 18.370742337723765, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.38826751708984375, + "mean_abs_diff": 0.0642109364271164, + "cosine_similarity": 0.9998274445533752, + "kl_divergence": 0.0006867424058280134, + "sigma_level": 49.15437011428163, + "cpk": 16.121769360216955, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.37309741973876953, + "mean_abs_diff": 0.06227777898311615, + "cosine_similarity": 0.9998220205307007, + "kl_divergence": 0.0021385287649098024, + "sigma_level": 50.534553708367916, + "cpk": 16.58258625555096, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 18589, + "gpu_argmax": 18589, + "max_abs_diff": 0.3214547634124756, + "mean_abs_diff": 0.053083546459674835, + "cosine_similarity": 0.9998363256454468, + "kl_divergence": 0.000752778125991802, + "sigma_level": 59.39349988162695, + "cpk": 19.535098493011564, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.36084842681884766, + "mean_abs_diff": 0.061356306076049805, + "cosine_similarity": 0.9998142123222351, + "kl_divergence": 0.003312728169157689, + "sigma_level": 51.05759445120621, + "cpk": 16.75813936768082, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 7354, + "gpu_argmax": 7354, + "max_abs_diff": 0.44296157360076904, + "mean_abs_diff": 0.06914495676755905, + "cosine_similarity": 0.9997162818908691, + "kl_divergence": 0.0032034377972692007, + "sigma_level": 46.07774351870567, + "cpk": 15.093744207606258, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.4447762966156006, + "mean_abs_diff": 0.07214333862066269, + "cosine_similarity": 0.999789834022522, + "kl_divergence": 0.002128935912980073, + "sigma_level": 44.064428985081875, + "cpk": 14.42323007657755, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1033, + "gpu_argmax": 1033, + "max_abs_diff": 0.43563520908355713, + "mean_abs_diff": 0.07464952021837234, + "cosine_similarity": 0.9997342228889465, + "kl_divergence": 0.0023521526351508127, + "sigma_level": 42.89405481276757, + "cpk": 14.031183219923053, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.41919422149658203, + "mean_abs_diff": 0.07205758988857269, + "cosine_similarity": 0.9996944665908813, + "kl_divergence": 0.0015906886315562518, + "sigma_level": 44.11487020371673, + "cpk": 14.44005579914499, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.4340832233428955, + "mean_abs_diff": 0.06832505017518997, + "cosine_similarity": 0.9997145533561707, + "kl_divergence": 4.531155001990606e-05, + "sigma_level": 46.65343302362604, + "cpk": 15.285511161860004, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.5156793594360352, + "mean_abs_diff": 0.07876264303922653, + "cosine_similarity": 0.9997888803482056, + "kl_divergence": 0.002400534868525818, + "sigma_level": 40.82280172433786, + "cpk": 13.339657928106357, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.49131178855895996, + "mean_abs_diff": 0.06945641338825226, + "cosine_similarity": 0.9997475743293762, + "kl_divergence": 0.0013757893037735654, + "sigma_level": 45.35377881058699, + "cpk": 14.855417036046704, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 2390, + "gpu_argmax": 2390, + "max_abs_diff": 0.5032455325126648, + "mean_abs_diff": 0.07041440159082413, + "cosine_similarity": 0.9997410178184509, + "kl_divergence": 0.0025526453726194957, + "sigma_level": 44.967915760286665, + "cpk": 14.72543951350829, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.47355103492736816, + "mean_abs_diff": 0.08038556575775146, + "cosine_similarity": 0.999701976776123, + "kl_divergence": 0.0016015019282984591, + "sigma_level": 39.61497617254028, + "cpk": 12.939619368170968, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.4618046283721924, + "mean_abs_diff": 0.07349393516778946, + "cosine_similarity": 0.999594509601593, + "kl_divergence": 0.0007789350432452571, + "sigma_level": 43.43102772649833, + "cpk": 14.211016147499297, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 1865, + "max_abs_diff": 0.40026283264160156, + "mean_abs_diff": 0.06240736320614815, + "cosine_similarity": 0.9997082948684692, + "kl_divergence": 0.002672701304671866, + "sigma_level": 50.514450324264075, + "cpk": 16.57544397070925, + "verdict": "WarnArgmax" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.38044023513793945, + "mean_abs_diff": 0.06645609438419342, + "cosine_similarity": 0.9998005628585815, + "kl_divergence": 0.000939163318030381, + "sigma_level": 47.37136548013599, + "cpk": 15.528112165423995, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.4195518493652344, + "mean_abs_diff": 0.06238096207380295, + "cosine_similarity": 0.9997071623802185, + "kl_divergence": 0.0019081753538363014, + "sigma_level": 50.68498889665763, + "cpk": 16.631514768046415, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json index 386cde6b83..10d70e594d 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "c642576eecb62daa", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md heading '2026-09-06T13:5xZ' \u2014 the record states the hour and deliberately not the minute, so only the date is carried here", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9508274793624878, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "FAIL", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 15, - "gpu_argmax": 16, - "max_abs_diff": 11.973122596740723, - "mean_abs_diff": 1.4562522172927856, - "cosine_similarity": 0.9508274793624878, - "kl_divergence": 5.417725371363736, - "sigma_level": 2.14457759095467, - "cpk": 0.4546053743195434, - "verdict": "WarnOutOfSpec" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.4650428295135498, - "mean_abs_diff": 0.07376201450824738, - "cosine_similarity": 0.9998178482055664, - "kl_divergence": 0.000053598414848508557, - "sigma_level": 42.80674637067787, - "cpk": 14.005789469655557, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.3866511583328247, - "mean_abs_diff": 0.06590328365564346, - "cosine_similarity": 0.9998971819877625, - "kl_divergence": 6.680759951875907e-6, - "sigma_level": 47.80867514885475, - "cpk": 15.67366265965696, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.3977069854736328, - "mean_abs_diff": 0.05706659331917763, - "cosine_similarity": 0.9998915195465088, - "kl_divergence": 0.0000789326202787771, - "sigma_level": 54.87129607627002, - "cpk": 18.029488863916615, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.34780454635620117, - "mean_abs_diff": 0.05530937761068344, - "cosine_similarity": 0.9998860359191895, - "kl_divergence": 7.848848746470546e-6, - "sigma_level": 56.51914696416572, - "cpk": 18.579212417915677, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3383975028991699, - "mean_abs_diff": 0.05794849619269371, - "cosine_similarity": 0.9998785853385925, - "kl_divergence": 0.00012155244184822574, - "sigma_level": 55.224400447456055, - "cpk": 18.14145256922925, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.3957533836364746, - "mean_abs_diff": 0.06864283233880997, - "cosine_similarity": 0.9999083876609802, - "kl_divergence": 6.281946350910951e-6, - "sigma_level": 46.65570749970518, - "cpk": 15.28502084093916, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.4882650375366211, - "mean_abs_diff": 0.0651940405368805, - "cosine_similarity": 0.9999063014984131, - "kl_divergence": 0.000016794876271664552, - "sigma_level": 47.99269977987777, - "cpk": 15.73683009204895, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2981089949607849, - "mean_abs_diff": 0.05331605300307274, - "cosine_similarity": 0.9998162388801575, - "kl_divergence": 0.0004707982680592144, - "sigma_level": 59.65800432330056, - "cpk": 19.620940664386968, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.3762543201446533, - "mean_abs_diff": 0.056634485721588135, - "cosine_similarity": 0.9998602271080017, - "kl_divergence": 0.00200288227214394, - "sigma_level": 55.89523235415072, - "cpk": 18.367944306494742, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.3492332696914673, - "mean_abs_diff": 0.05141938105225563, - "cosine_similarity": 0.9999210238456726, - "kl_divergence": 0.002701770685362771, - "sigma_level": 60.75193355286236, - "cpk": 19.99032561586946, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.39516687393188477, - "mean_abs_diff": 0.06365680694580078, - "cosine_similarity": 0.9998645186424255, - "kl_divergence": 0.0017848021580618698, - "sigma_level": 50.93329113486925, - "cpk": 16.70757615488253, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.34313130378723145, - "mean_abs_diff": 0.04843280836939812, - "cosine_similarity": 0.999874472618103, - "kl_divergence": 0.0004959427385190338, - "sigma_level": 64.8719807942433, - "cpk": 21.362165913551863, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.33470678329467773, - "mean_abs_diff": 0.052488118410110474, - "cosine_similarity": 0.9998583793640137, - "kl_divergence": 0.0016068498758497576, - "sigma_level": 59.61238325877218, - "cpk": 19.610049266991116, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.3048872947692871, - "mean_abs_diff": 0.049873605370521545, - "cosine_similarity": 0.9998965859413147, - "kl_divergence": 0.0029217986943148653, - "sigma_level": 62.751582374651335, - "cpk": 20.656390153573017, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.4006218910217285, - "mean_abs_diff": 0.05147523805499077, - "cosine_similarity": 0.9998210072517395, - "kl_divergence": 0.0006769849863572717, - "sigma_level": 61.15841057435974, - "cpk": 20.123791546171578, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3487355709075928, - "mean_abs_diff": 0.05283060669898987, - "cosine_similarity": 0.9998836517333984, - "kl_divergence": 0.0022685891938408463, - "sigma_level": 59.646889061218864, - "cpk": 19.61969790900533, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.3330197334289551, - "mean_abs_diff": 0.052766453474760056, - "cosine_similarity": 0.9998737573623657, - "kl_divergence": 0.00006840857161954282, - "sigma_level": 59.305164302653225, - "cpk": 19.50761116813533, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.316272497177124, - "mean_abs_diff": 0.051003385335206985, - "cosine_similarity": 0.9998650550842285, - "kl_divergence": 0.000382817233105055, - "sigma_level": 61.19029400286672, - "cpk": 20.13668865563867, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3347015380859375, - "mean_abs_diff": 0.05140787735581398, - "cosine_similarity": 0.9998480081558228, - "kl_divergence": 0.0014383470256046078, - "sigma_level": 61.62469731210393, - "cpk": 20.277566197242162, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.3056960105895996, - "mean_abs_diff": 0.04827465862035751, - "cosine_similarity": 0.9998608231544495, - "kl_divergence": 0.0018885915278280133, - "sigma_level": 64.61486353567884, - "cpk": 21.278349471977457, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.6572532653808594, - "mean_abs_diff": 0.10043209791183472, - "cosine_similarity": 0.9996307492256165, - "kl_divergence": 0.005978000238223978, - "sigma_level": 31.087704271296346, - "cpk": 10.102384477163033, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.5020737648010254, - "mean_abs_diff": 0.07753492891788483, - "cosine_similarity": 0.9997056126594543, - "kl_divergence": 0.0061177710560870804, - "sigma_level": 40.33543045360021, - "cpk": 13.184526423442389, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3872443437576294, - "mean_abs_diff": 0.062455497682094574, - "cosine_similarity": 0.9998182654380798, - "kl_divergence": 0.0008687540237392052, - "sigma_level": 50.5086856784924, - "cpk": 16.57334980105424, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.3850289583206177, - "mean_abs_diff": 0.06431932002305984, - "cosine_similarity": 0.9998119473457336, - "kl_divergence": 0.002553620620018977, - "sigma_level": 49.01901527706649, - "cpk": 16.076932614787086, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3047807812690735, - "mean_abs_diff": 0.04829900339245796, - "cosine_similarity": 0.999798059463501, - "kl_divergence": 0.00021319612052604753, - "sigma_level": 66.28606747309414, - "cpk": 21.828559907885072, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3059554100036621, - "mean_abs_diff": 0.048369839787483215, - "cosine_similarity": 0.9998838901519775, - "kl_divergence": 0.0006849703836857524, - "sigma_level": 65.17807355881911, - "cpk": 21.463303438298297, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.41999053955078125, - "mean_abs_diff": 0.07317500561475754, - "cosine_similarity": 0.9994683265686035, - "kl_divergence": 0.0009813839538431752, - "sigma_level": 43.85812919626951, - "cpk": 14.351933161240689, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.35340678691864014, - "mean_abs_diff": 0.056793734431266785, - "cosine_similarity": 0.999854326248169, - "kl_divergence": 0.0038247994839233756, - "sigma_level": 55.143344205449615, - "cpk": 18.120131697945187, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.35067176818847656, - "mean_abs_diff": 0.0629408061504364, - "cosine_similarity": 0.9997932314872742, - "kl_divergence": 0.0006541521126937118, - "sigma_level": 50.225147145014844, - "cpk": 16.478281444143988, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.29090166091918945, - "mean_abs_diff": 0.04321891814470291, - "cosine_similarity": 0.9998005628585815, - "kl_divergence": 0.00004718849190647301, - "sigma_level": 72.6417500127864, - "cpk": 23.952291850287917, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2871342897415161, - "mean_abs_diff": 0.044014591723680496, - "cosine_similarity": 0.9999145865440369, - "kl_divergence": 0.0004054254268827301, - "sigma_level": 71.05546670016032, - "cpk": 23.424532453674846, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.280792236328125, - "mean_abs_diff": 0.045980505645275116, - "cosine_similarity": 0.9999239444732666, - "kl_divergence": 0.001747060931358211, - "sigma_level": 68.52831234877785, - "cpk": 22.580190245191435, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 1075, - "max_abs_diff": 0.2834291458129883, - "mean_abs_diff": 0.04462064057588577, - "cosine_similarity": 0.9999250769615173, - "kl_divergence": 0.0019013842820190484, - "sigma_level": 70.39793470719678, - "cpk": 23.204211490577716, - "verdict": "WarnArgmax" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.31037187576293945, - "mean_abs_diff": 0.052936043590307236, - "cosine_similarity": 0.9998127222061157, - "kl_divergence": 0.00005010117795112397, - "sigma_level": 61.08032036537772, - "cpk": 20.090660913344944, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.35306501388549805, - "mean_abs_diff": 0.05245150998234749, - "cosine_similarity": 0.9999088644981384, - "kl_divergence": 0.0014205977063074006, - "sigma_level": 59.50700084759674, - "cpk": 19.575564278450805, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.4437136650085449, - "mean_abs_diff": 0.0651242807507515, - "cosine_similarity": 0.999549925327301, - "kl_divergence": 0.002034610592857552, - "sigma_level": 47.985960776301, - "cpk": 15.734899326959468, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.2869229316711426, - "mean_abs_diff": 0.04254760593175888, - "cosine_similarity": 0.9998387098312378, - "kl_divergence": 0.00012317854697163394, - "sigma_level": 73.4563462456492, - "cpk": 24.22499944244584, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.31911468505859375, - "mean_abs_diff": 0.04899978265166283, - "cosine_similarity": 0.9999270439147949, - "kl_divergence": 0.001279600249078306, - "sigma_level": 64.5414262336222, - "cpk": 21.25026575641775, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3646974563598633, - "mean_abs_diff": 0.07297120988368988, - "cosine_similarity": 0.9998273253440857, - "kl_divergence": 0.001634865324591819, - "sigma_level": 45.15382628552608, - "cpk": 14.77669798393096, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.3667411804199219, - "mean_abs_diff": 0.07524772733449936, - "cosine_similarity": 0.9998779892921448, - "kl_divergence": 0.001175862369518911, - "sigma_level": 44.43649313589075, - "cpk": 14.533518952031011, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.28829050064086914, - "mean_abs_diff": 0.0564601756632328, - "cosine_similarity": 0.9999026656150818, - "kl_divergence": 0.0012793447849670605, - "sigma_level": 57.344280001367196, - "cpk": 18.844954323609166, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.2895240783691406, - "mean_abs_diff": 0.045262835919857025, - "cosine_similarity": 0.999929666519165, - "kl_divergence": 0.0014526099267751793, - "sigma_level": 69.36425130160303, - "cpk": 22.859781873420328, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.30231142044067383, - "mean_abs_diff": 0.04469147324562073, - "cosine_similarity": 0.9998422265052795, - "kl_divergence": 0.0002661089801215439, - "sigma_level": 71.27346144399971, - "cpk": 23.492377481729296, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.31220388412475586, - "mean_abs_diff": 0.0626152977347374, - "cosine_similarity": 0.9998358488082886, - "kl_divergence": 0.0012580873445873658, - "sigma_level": 52.37533010732988, - "cpk": 17.185151961724493, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.3172950744628906, - "mean_abs_diff": 0.056548990309238434, - "cosine_similarity": 0.9998992085456848, - "kl_divergence": 0.0019092303061413402, - "sigma_level": 57.02370802279627, - "cpk": 18.73918324823393, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.287722110748291, - "mean_abs_diff": 0.046183329075574875, - "cosine_similarity": 0.9998865127563477, - "kl_divergence": 0.001074622186835361, - "sigma_level": 67.91322827569736, - "cpk": 22.376371177729023, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.28690052032470703, - "mean_abs_diff": 0.04460417479276657, - "cosine_similarity": 0.9998990297317505, - "kl_divergence": 0.001637326459726193, - "sigma_level": 70.32867329867013, - "cpk": 23.181478396493603, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.2437753677368164, - "mean_abs_diff": 0.042050521820783615, - "cosine_similarity": 0.9999042749404907, - "kl_divergence": 0.0012921399977282274, - "sigma_level": 75.96778021835588, - "cpk": 25.056386339472922, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.26329851150512695, - "mean_abs_diff": 0.04378020763397217, - "cosine_similarity": 0.9998823404312134, - "kl_divergence": 0.00045292527632992516, - "sigma_level": 72.18207270711018, - "cpk": 23.79734539148941, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.3285568952560425, - "mean_abs_diff": 0.04826463386416435, - "cosine_similarity": 0.9999091625213623, - "kl_divergence": 0.003169107297679868, - "sigma_level": 64.72904795942044, - "cpk": 21.316005669795366, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 1030, - "gpu_argmax": 1030, - "max_abs_diff": 0.2997932434082031, - "mean_abs_diff": 0.05100039765238762, - "cosine_similarity": 0.9998959302902222, - "kl_divergence": 0.0007471735698977093, - "sigma_level": 62.57656831041338, - "cpk": 20.592903614508387, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.35276174545288086, - "mean_abs_diff": 0.054895516484975815, - "cosine_similarity": 0.9999022483825684, - "kl_divergence": 0.0010573862584377517, - "sigma_level": 58.83785079955426, - "cpk": 19.343455749142464, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.23547744750976562, - "mean_abs_diff": 0.03824745863676071, - "cosine_similarity": 0.9999030828475952, - "kl_divergence": 0.00013315070766980216, - "sigma_level": 82.05800083254637, - "cpk": 27.091124444793948, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.23178386688232422, - "mean_abs_diff": 0.0405409075319767, - "cosine_similarity": 0.9998529553413391, - "kl_divergence": 0.00007050667318571868, - "sigma_level": 78.69024316368039, - "cpk": 25.96423323191284, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.2403573989868164, - "mean_abs_diff": 0.04604782164096832, - "cosine_similarity": 0.9998740553855896, - "kl_divergence": 0.0012601311136737736, - "sigma_level": 69.5124300500558, - "cpk": 22.904068684953998, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.2761220932006836, - "mean_abs_diff": 0.046715147793293, - "cosine_similarity": 0.9999170303344727, - "kl_divergence": 0.0011468693795866846, - "sigma_level": 68.6136821228989, - "cpk": 22.604119182548526, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.32673943042755127, - "mean_abs_diff": 0.042964544147253036, - "cosine_similarity": 0.999886691570282, - "kl_divergence": 0.0004910417376588438, - "sigma_level": 72.59800358871821, - "cpk": 23.939406185390244, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.2835589647293091, - "mean_abs_diff": 0.042426157742738724, - "cosine_similarity": 0.9998935461044312, - "kl_divergence": 0.00015988472627614193, - "sigma_level": 73.71183167144353, - "cpk": 24.310001407314605, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.26973533630371094, - "mean_abs_diff": 0.04785580188035965, - "cosine_similarity": 0.9999091625213623, - "kl_divergence": 0.00010123936424381408, - "sigma_level": 67.38781163452288, - "cpk": 22.193862397946567, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.28846168518066406, - "mean_abs_diff": 0.04364936426281929, - "cosine_similarity": 0.9999343156814575, - "kl_divergence": 0.0013821431895069671, - "sigma_level": 72.21723594234979, - "cpk": 23.809725610974798, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.26705145835876465, - "mean_abs_diff": 0.045417461544275284, - "cosine_similarity": 0.9998982548713684, - "kl_divergence": 0.00017813759490774671, - "sigma_level": 70.84042781016798, - "cpk": 23.34535990290197, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.3231534957885742, - "mean_abs_diff": 0.06047097593545914, - "cosine_similarity": 0.9998999834060669, - "kl_divergence": 0.0016757824665770912, - "sigma_level": 53.974261257017965, - "cpk": 17.719430731205378, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2636291980743408, - "mean_abs_diff": 0.04413612186908722, - "cosine_similarity": 0.9998951554298401, - "kl_divergence": 0.00045192088792373277, - "sigma_level": 71.99334230002293, - "cpk": 23.732988522547917, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2389669418334961, - "mean_abs_diff": 0.04086123779416084, - "cosine_similarity": 0.9999276995658875, - "kl_divergence": 0.0008661659413689716, - "sigma_level": 76.81578693588703, - "cpk": 25.343696633934606, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.25644922256469727, - "mean_abs_diff": 0.04367988184094429, - "cosine_similarity": 0.9998958706855774, - "kl_divergence": 0.001030973317421363, - "sigma_level": 71.68527909970481, - "cpk": 23.63415932316742, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.26444530487060547, - "mean_abs_diff": 0.05307325720787048, - "cosine_similarity": 0.9998828172683716, - "kl_divergence": 0.00022194838380929847, - "sigma_level": 61.76488152384485, - "cpk": 20.31512188765423, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.3115396499633789, - "mean_abs_diff": 0.054525721818208694, - "cosine_similarity": 0.9999011754989624, - "kl_divergence": 0.0005107738590351378, - "sigma_level": 59.64675652494656, - "cpk": 19.61122863717905, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.2833176851272583, - "mean_abs_diff": 0.0467853918671608, - "cosine_similarity": 0.9998739361763, - "kl_divergence": 0.0015430497955592104, - "sigma_level": 67.90594397032366, - "cpk": 22.370564140211133, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2992556095123291, - "mean_abs_diff": 0.05715445801615715, - "cosine_similarity": 0.9998778104782104, - "kl_divergence": 4.221885622172819e-6, - "sigma_level": 57.44807291346208, - "cpk": 18.875739848533893, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.24626314640045166, - "mean_abs_diff": 0.04180094972252846, - "cosine_similarity": 0.9998927712440491, - "kl_divergence": 0.00018937640439664716, - "sigma_level": 74.82955276570739, - "cpk": 24.68252205749258, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.3256549835205078, - "mean_abs_diff": 0.05863361805677414, - "cosine_similarity": 0.9999057650566101, - "kl_divergence": 0.0011160061469011708, - "sigma_level": 55.66880028635367, - "cpk": 18.284261497645485, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.40552783012390137, - "mean_abs_diff": 0.06164288520812988, - "cosine_similarity": 0.9998986721038818, - "kl_divergence": 0.0016408893623984504, - "sigma_level": 52.06927285767559, - "cpk": 17.088949268422155, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.30518674850463867, - "mean_abs_diff": 0.052556075155735016, - "cosine_similarity": 0.9998877048492432, - "kl_divergence": 0.0011176506364210586, - "sigma_level": 60.6323717384759, - "cpk": 19.945240622328814, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2461402416229248, - "mean_abs_diff": 0.04279375076293945, - "cosine_similarity": 0.9999181628227234, - "kl_divergence": 0.00028949609438141944, - "sigma_level": 73.38914463933241, - "cpk": 24.201331816077396, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.28456664085388184, - "mean_abs_diff": 0.044448915868997574, - "cosine_similarity": 0.9999171495437622, - "kl_divergence": 0.0006934676863507526, - "sigma_level": 70.69053702108894, - "cpk": 23.301669195964262, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2627677917480469, - "mean_abs_diff": 0.04326142370700836, - "cosine_similarity": 0.9999173283576965, - "kl_divergence": 0.0003628763594607102, - "sigma_level": 73.39988060423036, - "cpk": 24.2020115901715, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.28603506088256836, - "mean_abs_diff": 0.04886169731616974, - "cosine_similarity": 0.9998109340667725, - "kl_divergence": 0.0013024145853326003, - "sigma_level": 64.2417199046864, - "cpk": 21.15232667880772, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 15, + "gpu_argmax": 16, + "max_abs_diff": 11.973122596740723, + "mean_abs_diff": 1.4562522172927856, + "cosine_similarity": 0.9508274793624878, + "kl_divergence": 5.417725371363736, + "sigma_level": 2.14457759095467, + "cpk": 0.4546053743195434, + "verdict": "WarnOutOfSpec" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.4650428295135498, + "mean_abs_diff": 0.07376201450824738, + "cosine_similarity": 0.9998178482055664, + "kl_divergence": 5.3598414848508557e-05, + "sigma_level": 42.80674637067787, + "cpk": 14.005789469655557, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.3866511583328247, + "mean_abs_diff": 0.06590328365564346, + "cosine_similarity": 0.9998971819877625, + "kl_divergence": 6.680759951875907e-06, + "sigma_level": 47.80867514885475, + "cpk": 15.67366265965696, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.3977069854736328, + "mean_abs_diff": 0.05706659331917763, + "cosine_similarity": 0.9998915195465088, + "kl_divergence": 7.89326202787771e-05, + "sigma_level": 54.87129607627002, + "cpk": 18.029488863916615, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.34780454635620117, + "mean_abs_diff": 0.05530937761068344, + "cosine_similarity": 0.9998860359191895, + "kl_divergence": 7.848848746470546e-06, + "sigma_level": 56.51914696416572, + "cpk": 18.579212417915677, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3383975028991699, + "mean_abs_diff": 0.05794849619269371, + "cosine_similarity": 0.9998785853385925, + "kl_divergence": 0.00012155244184822574, + "sigma_level": 55.224400447456055, + "cpk": 18.14145256922925, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.3957533836364746, + "mean_abs_diff": 0.06864283233880997, + "cosine_similarity": 0.9999083876609802, + "kl_divergence": 6.281946350910951e-06, + "sigma_level": 46.65570749970518, + "cpk": 15.28502084093916, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.4882650375366211, + "mean_abs_diff": 0.0651940405368805, + "cosine_similarity": 0.9999063014984131, + "kl_divergence": 1.6794876271664552e-05, + "sigma_level": 47.99269977987777, + "cpk": 15.73683009204895, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2981089949607849, + "mean_abs_diff": 0.05331605300307274, + "cosine_similarity": 0.9998162388801575, + "kl_divergence": 0.0004707982680592144, + "sigma_level": 59.65800432330056, + "cpk": 19.620940664386968, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.3762543201446533, + "mean_abs_diff": 0.056634485721588135, + "cosine_similarity": 0.9998602271080017, + "kl_divergence": 0.00200288227214394, + "sigma_level": 55.89523235415072, + "cpk": 18.367944306494742, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.3492332696914673, + "mean_abs_diff": 0.05141938105225563, + "cosine_similarity": 0.9999210238456726, + "kl_divergence": 0.002701770685362771, + "sigma_level": 60.75193355286236, + "cpk": 19.99032561586946, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.39516687393188477, + "mean_abs_diff": 0.06365680694580078, + "cosine_similarity": 0.9998645186424255, + "kl_divergence": 0.0017848021580618698, + "sigma_level": 50.93329113486925, + "cpk": 16.70757615488253, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.34313130378723145, + "mean_abs_diff": 0.04843280836939812, + "cosine_similarity": 0.999874472618103, + "kl_divergence": 0.0004959427385190338, + "sigma_level": 64.8719807942433, + "cpk": 21.362165913551863, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.33470678329467773, + "mean_abs_diff": 0.052488118410110474, + "cosine_similarity": 0.9998583793640137, + "kl_divergence": 0.0016068498758497576, + "sigma_level": 59.61238325877218, + "cpk": 19.610049266991116, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.3048872947692871, + "mean_abs_diff": 0.049873605370521545, + "cosine_similarity": 0.9998965859413147, + "kl_divergence": 0.0029217986943148653, + "sigma_level": 62.751582374651335, + "cpk": 20.656390153573017, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.4006218910217285, + "mean_abs_diff": 0.05147523805499077, + "cosine_similarity": 0.9998210072517395, + "kl_divergence": 0.0006769849863572717, + "sigma_level": 61.15841057435974, + "cpk": 20.123791546171578, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3487355709075928, + "mean_abs_diff": 0.05283060669898987, + "cosine_similarity": 0.9998836517333984, + "kl_divergence": 0.0022685891938408463, + "sigma_level": 59.646889061218864, + "cpk": 19.61969790900533, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.3330197334289551, + "mean_abs_diff": 0.052766453474760056, + "cosine_similarity": 0.9998737573623657, + "kl_divergence": 6.840857161954282e-05, + "sigma_level": 59.305164302653225, + "cpk": 19.50761116813533, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.316272497177124, + "mean_abs_diff": 0.051003385335206985, + "cosine_similarity": 0.9998650550842285, + "kl_divergence": 0.000382817233105055, + "sigma_level": 61.19029400286672, + "cpk": 20.13668865563867, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3347015380859375, + "mean_abs_diff": 0.05140787735581398, + "cosine_similarity": 0.9998480081558228, + "kl_divergence": 0.0014383470256046078, + "sigma_level": 61.62469731210393, + "cpk": 20.277566197242162, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.3056960105895996, + "mean_abs_diff": 0.04827465862035751, + "cosine_similarity": 0.9998608231544495, + "kl_divergence": 0.0018885915278280133, + "sigma_level": 64.61486353567884, + "cpk": 21.278349471977457, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.6572532653808594, + "mean_abs_diff": 0.10043209791183472, + "cosine_similarity": 0.9996307492256165, + "kl_divergence": 0.005978000238223978, + "sigma_level": 31.087704271296346, + "cpk": 10.102384477163033, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.5020737648010254, + "mean_abs_diff": 0.07753492891788483, + "cosine_similarity": 0.9997056126594543, + "kl_divergence": 0.0061177710560870804, + "sigma_level": 40.33543045360021, + "cpk": 13.184526423442389, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3872443437576294, + "mean_abs_diff": 0.062455497682094574, + "cosine_similarity": 0.9998182654380798, + "kl_divergence": 0.0008687540237392052, + "sigma_level": 50.5086856784924, + "cpk": 16.57334980105424, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.3850289583206177, + "mean_abs_diff": 0.06431932002305984, + "cosine_similarity": 0.9998119473457336, + "kl_divergence": 0.002553620620018977, + "sigma_level": 49.01901527706649, + "cpk": 16.076932614787086, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3047807812690735, + "mean_abs_diff": 0.04829900339245796, + "cosine_similarity": 0.999798059463501, + "kl_divergence": 0.00021319612052604753, + "sigma_level": 66.28606747309414, + "cpk": 21.828559907885072, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3059554100036621, + "mean_abs_diff": 0.048369839787483215, + "cosine_similarity": 0.9998838901519775, + "kl_divergence": 0.0006849703836857524, + "sigma_level": 65.17807355881911, + "cpk": 21.463303438298297, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.41999053955078125, + "mean_abs_diff": 0.07317500561475754, + "cosine_similarity": 0.9994683265686035, + "kl_divergence": 0.0009813839538431752, + "sigma_level": 43.85812919626951, + "cpk": 14.351933161240689, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.35340678691864014, + "mean_abs_diff": 0.056793734431266785, + "cosine_similarity": 0.999854326248169, + "kl_divergence": 0.0038247994839233756, + "sigma_level": 55.143344205449615, + "cpk": 18.120131697945187, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.35067176818847656, + "mean_abs_diff": 0.0629408061504364, + "cosine_similarity": 0.9997932314872742, + "kl_divergence": 0.0006541521126937118, + "sigma_level": 50.225147145014844, + "cpk": 16.478281444143988, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.29090166091918945, + "mean_abs_diff": 0.04321891814470291, + "cosine_similarity": 0.9998005628585815, + "kl_divergence": 4.718849190647301e-05, + "sigma_level": 72.6417500127864, + "cpk": 23.952291850287917, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2871342897415161, + "mean_abs_diff": 0.044014591723680496, + "cosine_similarity": 0.9999145865440369, + "kl_divergence": 0.0004054254268827301, + "sigma_level": 71.05546670016032, + "cpk": 23.424532453674846, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.280792236328125, + "mean_abs_diff": 0.045980505645275116, + "cosine_similarity": 0.9999239444732666, + "kl_divergence": 0.001747060931358211, + "sigma_level": 68.52831234877785, + "cpk": 22.580190245191435, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 1075, + "max_abs_diff": 0.2834291458129883, + "mean_abs_diff": 0.04462064057588577, + "cosine_similarity": 0.9999250769615173, + "kl_divergence": 0.0019013842820190484, + "sigma_level": 70.39793470719678, + "cpk": 23.204211490577716, + "verdict": "WarnArgmax" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.31037187576293945, + "mean_abs_diff": 0.052936043590307236, + "cosine_similarity": 0.9998127222061157, + "kl_divergence": 5.010117795112397e-05, + "sigma_level": 61.08032036537772, + "cpk": 20.090660913344944, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.35306501388549805, + "mean_abs_diff": 0.05245150998234749, + "cosine_similarity": 0.9999088644981384, + "kl_divergence": 0.0014205977063074006, + "sigma_level": 59.50700084759674, + "cpk": 19.575564278450805, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.4437136650085449, + "mean_abs_diff": 0.0651242807507515, + "cosine_similarity": 0.999549925327301, + "kl_divergence": 0.002034610592857552, + "sigma_level": 47.985960776301, + "cpk": 15.734899326959468, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.2869229316711426, + "mean_abs_diff": 0.04254760593175888, + "cosine_similarity": 0.9998387098312378, + "kl_divergence": 0.00012317854697163394, + "sigma_level": 73.4563462456492, + "cpk": 24.22499944244584, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.31911468505859375, + "mean_abs_diff": 0.04899978265166283, + "cosine_similarity": 0.9999270439147949, + "kl_divergence": 0.001279600249078306, + "sigma_level": 64.5414262336222, + "cpk": 21.25026575641775, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3646974563598633, + "mean_abs_diff": 0.07297120988368988, + "cosine_similarity": 0.9998273253440857, + "kl_divergence": 0.001634865324591819, + "sigma_level": 45.15382628552608, + "cpk": 14.77669798393096, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.3667411804199219, + "mean_abs_diff": 0.07524772733449936, + "cosine_similarity": 0.9998779892921448, + "kl_divergence": 0.001175862369518911, + "sigma_level": 44.43649313589075, + "cpk": 14.533518952031011, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.28829050064086914, + "mean_abs_diff": 0.0564601756632328, + "cosine_similarity": 0.9999026656150818, + "kl_divergence": 0.0012793447849670605, + "sigma_level": 57.344280001367196, + "cpk": 18.844954323609166, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.2895240783691406, + "mean_abs_diff": 0.045262835919857025, + "cosine_similarity": 0.999929666519165, + "kl_divergence": 0.0014526099267751793, + "sigma_level": 69.36425130160303, + "cpk": 22.859781873420328, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.30231142044067383, + "mean_abs_diff": 0.04469147324562073, + "cosine_similarity": 0.9998422265052795, + "kl_divergence": 0.0002661089801215439, + "sigma_level": 71.27346144399971, + "cpk": 23.492377481729296, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.31220388412475586, + "mean_abs_diff": 0.0626152977347374, + "cosine_similarity": 0.9998358488082886, + "kl_divergence": 0.0012580873445873658, + "sigma_level": 52.37533010732988, + "cpk": 17.185151961724493, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.3172950744628906, + "mean_abs_diff": 0.056548990309238434, + "cosine_similarity": 0.9998992085456848, + "kl_divergence": 0.0019092303061413402, + "sigma_level": 57.02370802279627, + "cpk": 18.73918324823393, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.287722110748291, + "mean_abs_diff": 0.046183329075574875, + "cosine_similarity": 0.9998865127563477, + "kl_divergence": 0.001074622186835361, + "sigma_level": 67.91322827569736, + "cpk": 22.376371177729023, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.28690052032470703, + "mean_abs_diff": 0.04460417479276657, + "cosine_similarity": 0.9998990297317505, + "kl_divergence": 0.001637326459726193, + "sigma_level": 70.32867329867013, + "cpk": 23.181478396493603, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.2437753677368164, + "mean_abs_diff": 0.042050521820783615, + "cosine_similarity": 0.9999042749404907, + "kl_divergence": 0.0012921399977282274, + "sigma_level": 75.96778021835588, + "cpk": 25.056386339472922, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.26329851150512695, + "mean_abs_diff": 0.04378020763397217, + "cosine_similarity": 0.9998823404312134, + "kl_divergence": 0.00045292527632992516, + "sigma_level": 72.18207270711018, + "cpk": 23.79734539148941, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.3285568952560425, + "mean_abs_diff": 0.04826463386416435, + "cosine_similarity": 0.9999091625213623, + "kl_divergence": 0.003169107297679868, + "sigma_level": 64.72904795942044, + "cpk": 21.316005669795366, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 1030, + "gpu_argmax": 1030, + "max_abs_diff": 0.2997932434082031, + "mean_abs_diff": 0.05100039765238762, + "cosine_similarity": 0.9998959302902222, + "kl_divergence": 0.0007471735698977093, + "sigma_level": 62.57656831041338, + "cpk": 20.592903614508387, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.35276174545288086, + "mean_abs_diff": 0.054895516484975815, + "cosine_similarity": 0.9999022483825684, + "kl_divergence": 0.0010573862584377517, + "sigma_level": 58.83785079955426, + "cpk": 19.343455749142464, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.23547744750976562, + "mean_abs_diff": 0.03824745863676071, + "cosine_similarity": 0.9999030828475952, + "kl_divergence": 0.00013315070766980216, + "sigma_level": 82.05800083254637, + "cpk": 27.091124444793948, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.23178386688232422, + "mean_abs_diff": 0.0405409075319767, + "cosine_similarity": 0.9998529553413391, + "kl_divergence": 7.050667318571868e-05, + "sigma_level": 78.69024316368039, + "cpk": 25.96423323191284, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.2403573989868164, + "mean_abs_diff": 0.04604782164096832, + "cosine_similarity": 0.9998740553855896, + "kl_divergence": 0.0012601311136737736, + "sigma_level": 69.5124300500558, + "cpk": 22.904068684953998, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.2761220932006836, + "mean_abs_diff": 0.046715147793293, + "cosine_similarity": 0.9999170303344727, + "kl_divergence": 0.0011468693795866846, + "sigma_level": 68.6136821228989, + "cpk": 22.604119182548526, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.32673943042755127, + "mean_abs_diff": 0.042964544147253036, + "cosine_similarity": 0.999886691570282, + "kl_divergence": 0.0004910417376588438, + "sigma_level": 72.59800358871821, + "cpk": 23.939406185390244, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.2835589647293091, + "mean_abs_diff": 0.042426157742738724, + "cosine_similarity": 0.9998935461044312, + "kl_divergence": 0.00015988472627614193, + "sigma_level": 73.71183167144353, + "cpk": 24.310001407314605, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.26973533630371094, + "mean_abs_diff": 0.04785580188035965, + "cosine_similarity": 0.9999091625213623, + "kl_divergence": 0.00010123936424381408, + "sigma_level": 67.38781163452288, + "cpk": 22.193862397946567, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.28846168518066406, + "mean_abs_diff": 0.04364936426281929, + "cosine_similarity": 0.9999343156814575, + "kl_divergence": 0.0013821431895069671, + "sigma_level": 72.21723594234979, + "cpk": 23.809725610974798, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.26705145835876465, + "mean_abs_diff": 0.045417461544275284, + "cosine_similarity": 0.9998982548713684, + "kl_divergence": 0.00017813759490774671, + "sigma_level": 70.84042781016798, + "cpk": 23.34535990290197, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.3231534957885742, + "mean_abs_diff": 0.06047097593545914, + "cosine_similarity": 0.9998999834060669, + "kl_divergence": 0.0016757824665770912, + "sigma_level": 53.974261257017965, + "cpk": 17.719430731205378, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2636291980743408, + "mean_abs_diff": 0.04413612186908722, + "cosine_similarity": 0.9998951554298401, + "kl_divergence": 0.00045192088792373277, + "sigma_level": 71.99334230002293, + "cpk": 23.732988522547917, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2389669418334961, + "mean_abs_diff": 0.04086123779416084, + "cosine_similarity": 0.9999276995658875, + "kl_divergence": 0.0008661659413689716, + "sigma_level": 76.81578693588703, + "cpk": 25.343696633934606, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.25644922256469727, + "mean_abs_diff": 0.04367988184094429, + "cosine_similarity": 0.9998958706855774, + "kl_divergence": 0.001030973317421363, + "sigma_level": 71.68527909970481, + "cpk": 23.63415932316742, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.26444530487060547, + "mean_abs_diff": 0.05307325720787048, + "cosine_similarity": 0.9998828172683716, + "kl_divergence": 0.00022194838380929847, + "sigma_level": 61.76488152384485, + "cpk": 20.31512188765423, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.3115396499633789, + "mean_abs_diff": 0.054525721818208694, + "cosine_similarity": 0.9999011754989624, + "kl_divergence": 0.0005107738590351378, + "sigma_level": 59.64675652494656, + "cpk": 19.61122863717905, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.2833176851272583, + "mean_abs_diff": 0.0467853918671608, + "cosine_similarity": 0.9998739361763, + "kl_divergence": 0.0015430497955592104, + "sigma_level": 67.90594397032366, + "cpk": 22.370564140211133, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2992556095123291, + "mean_abs_diff": 0.05715445801615715, + "cosine_similarity": 0.9998778104782104, + "kl_divergence": 4.221885622172819e-06, + "sigma_level": 57.44807291346208, + "cpk": 18.875739848533893, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.24626314640045166, + "mean_abs_diff": 0.04180094972252846, + "cosine_similarity": 0.9998927712440491, + "kl_divergence": 0.00018937640439664716, + "sigma_level": 74.82955276570739, + "cpk": 24.68252205749258, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.3256549835205078, + "mean_abs_diff": 0.05863361805677414, + "cosine_similarity": 0.9999057650566101, + "kl_divergence": 0.0011160061469011708, + "sigma_level": 55.66880028635367, + "cpk": 18.284261497645485, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.40552783012390137, + "mean_abs_diff": 0.06164288520812988, + "cosine_similarity": 0.9998986721038818, + "kl_divergence": 0.0016408893623984504, + "sigma_level": 52.06927285767559, + "cpk": 17.088949268422155, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.30518674850463867, + "mean_abs_diff": 0.052556075155735016, + "cosine_similarity": 0.9998877048492432, + "kl_divergence": 0.0011176506364210586, + "sigma_level": 60.6323717384759, + "cpk": 19.945240622328814, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2461402416229248, + "mean_abs_diff": 0.04279375076293945, + "cosine_similarity": 0.9999181628227234, + "kl_divergence": 0.00028949609438141944, + "sigma_level": 73.38914463933241, + "cpk": 24.201331816077396, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.28456664085388184, + "mean_abs_diff": 0.044448915868997574, + "cosine_similarity": 0.9999171495437622, + "kl_divergence": 0.0006934676863507526, + "sigma_level": 70.69053702108894, + "cpk": 23.301669195964262, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2627677917480469, + "mean_abs_diff": 0.04326142370700836, + "cosine_similarity": 0.9999173283576965, + "kl_divergence": 0.0003628763594607102, + "sigma_level": 73.39988060423036, + "cpk": 24.2020115901715, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.28603506088256836, + "mean_abs_diff": 0.04886169731616974, + "cosine_similarity": 0.9998109340667725, + "kl_divergence": 0.0013024145853326003, + "sigma_level": 64.2417199046864, + "cpk": 21.15232667880772, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json index 2e061ff232..1be80c540f 100644 --- a/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1/lambda/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "noah-Lambda-Vector", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-06", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-06. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1/lambda/RECORD.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "c642576eecb62daa", + "gpu": "NVIDIA GeForce RTX 4090", + "arch": "x86_64", + "sm": "89", + "generated_at_basis": "evidence/parity/l0-1/lambda/RECORD.md heading '2026-09-06T13:5xZ' \u2014 the record states the hour and deliberately not the minute, so only the date is carried here", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9986071586608887, + "min_cosine_position": 0, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.7765593528747559, - "mean_abs_diff": 0.11612559109926224, - "cosine_similarity": 0.9986071586608887, - "kl_divergence": 0.011310374807126194, - "sigma_level": 27.6123564403692, - "cpk": 8.936910379016284, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.21668052673339844, - "mean_abs_diff": 0.03283289819955826, - "cosine_similarity": 0.9998907446861267, - "kl_divergence": 6.3402314768562945e-6, - "sigma_level": 96.19123042846297, - "cpk": 31.800557069791992, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.21230220794677734, - "mean_abs_diff": 0.036253008991479874, - "cosine_similarity": 0.9999125599861145, - "kl_divergence": 1.2138987899670097e-6, - "sigma_level": 87.78216091036128, - "cpk": 28.99552301438919, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.22992169857025146, - "mean_abs_diff": 0.031362131237983704, - "cosine_similarity": 0.9999329447746277, - "kl_divergence": 0.00009697837023952451, - "sigma_level": 99.8193545508211, - "cpk": 33.01223920881423, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.20900297164916992, - "mean_abs_diff": 0.03212369233369827, - "cosine_similarity": 0.9999116063117981, - "kl_divergence": 1.2130307390144058e-6, - "sigma_level": 97.33557518850837, - "cpk": 32.184626890296194, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2239065170288086, - "mean_abs_diff": 0.033580731600522995, - "cosine_similarity": 0.9999029040336609, - "kl_divergence": 0.00002739006113745167, - "sigma_level": 95.1876333412705, - "cpk": 31.462838583179984, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.26349353790283203, - "mean_abs_diff": 0.037349067628383636, - "cosine_similarity": 0.9999385476112366, - "kl_divergence": 6.901890175435653e-7, - "sigma_level": 85.80957537462608, - "cpk": 28.336116155389544, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.19062137603759766, - "mean_abs_diff": 0.03048071265220642, - "cosine_similarity": 0.9999557137489319, - "kl_divergence": 8.014137110110945e-7, - "sigma_level": 103.11592767396162, - "cpk": 34.11005531121256, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18280649185180664, - "mean_abs_diff": 0.02944883517920971, - "cosine_similarity": 0.9999102354049683, - "kl_divergence": 0.00036753254509921214, - "sigma_level": 106.09218564707784, - "cpk": 35.10370427494904, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2662510871887207, - "mean_abs_diff": 0.03750442713499069, - "cosine_similarity": 0.9999082684516907, - "kl_divergence": 0.0008122461170267515, - "sigma_level": 83.32380704426187, - "cpk": 27.514184710595494, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.21428704261779785, - "mean_abs_diff": 0.03618254512548447, - "cosine_similarity": 0.9999296069145203, - "kl_divergence": 0.0013414333957104756, - "sigma_level": 86.32583699113798, - "cpk": 28.514988289343737, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.2245645523071289, - "mean_abs_diff": 0.03425215184688568, - "cosine_similarity": 0.9999250173568726, - "kl_divergence": 0.0005234989769493335, - "sigma_level": 91.05669514978527, - "cpk": 30.09232440418292, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.24261903762817383, - "mean_abs_diff": 0.032688990235328674, - "cosine_similarity": 0.9999192953109741, - "kl_divergence": 0.00023210232981628998, - "sigma_level": 96.43691504783891, - "cpk": 31.882936234752634, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.19068384170532227, - "mean_abs_diff": 0.029661044478416443, - "cosine_similarity": 0.9999234080314636, - "kl_divergence": 0.00042034598196102077, - "sigma_level": 106.34280606113548, - "cpk": 35.184748795333576, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.22031402587890625, - "mean_abs_diff": 0.03211377561092377, - "cosine_similarity": 0.9999381899833679, - "kl_divergence": 0.00032284577336933807, - "sigma_level": 97.25797222535725, - "cpk": 32.15904735041753, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.26050472259521484, - "mean_abs_diff": 0.05659817159175873, - "cosine_similarity": 0.9997627139091492, - "kl_divergence": 0.0003138751302377445, - "sigma_level": 60.00271273002659, - "cpk": 19.71790059075344, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2220613956451416, - "mean_abs_diff": 0.034589797258377075, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.0010091621728843534, - "sigma_level": 90.97388370911588, - "cpk": 30.06239722026317, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.2452479600906372, - "mean_abs_diff": 0.04691462218761444, - "cosine_similarity": 0.9998961687088013, - "kl_divergence": 0.000010286648444546095, - "sigma_level": 71.21310678671547, - "cpk": 23.45929092892974, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.20011234283447266, - "mean_abs_diff": 0.028810735791921616, - "cosine_similarity": 0.9999183416366577, - "kl_divergence": 0.0005886006120770999, - "sigma_level": 109.42638674224135, - "cpk": 36.21274102098917, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 9904, - "gpu_argmax": 5942, - "max_abs_diff": 0.2104625701904297, - "mean_abs_diff": 0.03633837774395943, - "cosine_similarity": 0.9999009966850281, - "kl_divergence": 0.000780974228869925, - "sigma_level": 89.61552988240601, - "cpk": 29.60046971275267, - "verdict": "WarnArgmax" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.2549419403076172, - "mean_abs_diff": 0.03461437672376633, - "cosine_similarity": 0.9998999238014221, - "kl_divergence": 0.0005821044683649174, - "sigma_level": 90.29548641907701, - "cpk": 29.838035307745198, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.2721090316772461, - "mean_abs_diff": 0.041589464992284775, - "cosine_similarity": 0.9999185800552368, - "kl_divergence": 0.0012391113766812047, - "sigma_level": 75.04357270930153, - "cpk": 24.754439066426386, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.23730623722076416, - "mean_abs_diff": 0.040260475128889084, - "cosine_similarity": 0.9998979568481445, - "kl_divergence": 0.0009176755486284543, - "sigma_level": 79.46140806613252, - "cpk": 26.22053985178142, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19652128219604492, - "mean_abs_diff": 0.032911960035562515, - "cosine_similarity": 0.9999001026153564, - "kl_divergence": 0.00011332451760925956, - "sigma_level": 95.68482644088411, - "cpk": 31.632510881642027, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.1758362054824829, - "mean_abs_diff": 0.029195617884397507, - "cosine_similarity": 0.9998802542686462, - "kl_divergence": 0.0006681351605656909, - "sigma_level": 106.90541285517136, - "cpk": 35.37504015309934, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.18475735187530518, - "mean_abs_diff": 0.02708299830555916, - "cosine_similarity": 0.9998446702957153, - "kl_divergence": 0.00006509554974005071, - "sigma_level": 115.87629249589668, - "cpk": 38.363907712522156, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.17609357833862305, - "mean_abs_diff": 0.027387676760554314, - "cosine_similarity": 0.9998769164085388, - "kl_divergence": 0.00018923296147374256, - "sigma_level": 114.58697406979748, - "cpk": 37.93413543936636, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.20337700843811035, - "mean_abs_diff": 0.0357307530939579, - "cosine_similarity": 0.9997212886810303, - "kl_divergence": 0.00068673561845549, - "sigma_level": 89.93727787207827, - "cpk": 29.7112987348936, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.2158222198486328, - "mean_abs_diff": 0.03296241909265518, - "cosine_similarity": 0.9998617768287659, - "kl_divergence": 0.0007174528587839378, - "sigma_level": 94.28973702732381, - "cpk": 31.170910856772032, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.18802005052566528, - "mean_abs_diff": 0.030701307579874992, - "cosine_similarity": 0.9998907446861267, - "kl_divergence": 0.000727208578105642, - "sigma_level": 102.51328926047806, - "cpk": 33.90882208477513, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.18789291381835938, - "mean_abs_diff": 0.028557559475302696, - "cosine_similarity": 0.9998390674591064, - "kl_divergence": 0.00007215432969221762, - "sigma_level": 109.68622544018605, - "cpk": 36.30104423784289, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.1830120086669922, - "mean_abs_diff": 0.029256917536258698, - "cosine_similarity": 0.9998962879180908, - "kl_divergence": 0.0002161959497020309, - "sigma_level": 107.87296541749743, - "cpk": 35.69465260136486, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.18589067459106445, - "mean_abs_diff": 0.030194438993930817, - "cosine_similarity": 0.9999088644981384, - "kl_divergence": 0.0004802196691873215, - "sigma_level": 105.23769924870247, - "cpk": 34.81443364208194, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.19057416915893555, - "mean_abs_diff": 0.030395982787013054, - "cosine_similarity": 0.9998965263366699, - "kl_divergence": 0.0008087840754308433, - "sigma_level": 103.21133882875218, - "cpk": 34.14234543637878, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.20451754331588745, - "mean_abs_diff": 0.03417585417628288, - "cosine_similarity": 0.999851405620575, - "kl_divergence": 0.000011874646119087019, - "sigma_level": 93.7446562083357, - "cpk": 30.98123509441341, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.2111678123474121, - "mean_abs_diff": 0.031797345727682114, - "cosine_similarity": 0.9999288320541382, - "kl_divergence": 0.0003358035896646284, - "sigma_level": 98.93926386784285, - "cpk": 32.71758745767694, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.2316608428955078, - "mean_abs_diff": 0.03618849068880081, - "cosine_similarity": 0.999769926071167, - "kl_divergence": 0.0009839003068092891, - "sigma_level": 87.81894152240396, - "cpk": 29.008144261835998, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.21175193786621094, - "mean_abs_diff": 0.03807161748409271, - "cosine_similarity": 0.9998552203178406, - "kl_divergence": 0.00012716871719756028, - "sigma_level": 85.63607322735568, - "cpk": 28.273665757222588, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.22519683837890625, - "mean_abs_diff": 0.032994192093610764, - "cosine_similarity": 0.9999074339866638, - "kl_divergence": 0.001043569292129262, - "sigma_level": 94.46420366663784, - "cpk": 31.228337048733696, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.19043952226638794, - "mean_abs_diff": 0.030219942331314087, - "cosine_similarity": 0.999907910823822, - "kl_divergence": 0.0013804227424616085, - "sigma_level": 104.09966497830833, - "cpk": 34.437731170073306, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.19405746459960938, - "mean_abs_diff": 0.03200043737888336, - "cosine_similarity": 0.9999300837516785, - "kl_divergence": 0.0005307637452940784, - "sigma_level": 99.51118657903375, - "cpk": 32.90502873512619, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2977466583251953, - "mean_abs_diff": 0.04491940513253212, - "cosine_similarity": 0.9999076128005981, - "kl_divergence": 0.0019453823155301797, - "sigma_level": 71.66086038536118, - "cpk": 23.61870652680407, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.20647287368774414, - "mean_abs_diff": 0.029774188995361328, - "cosine_similarity": 0.9999259114265442, - "kl_divergence": 0.0004456308198784806, - "sigma_level": 105.13497563493686, - "cpk": 34.784132825430845, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.17754626274108887, - "mean_abs_diff": 0.028160495683550835, - "cosine_similarity": 0.9998858571052551, - "kl_divergence": 0.00016564224134654518, - "sigma_level": 111.07898880573893, - "cpk": 36.765659653179874, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.37882566452026367, - "mean_abs_diff": 0.0732780396938324, - "cosine_similarity": 0.9998091459274292, - "kl_divergence": 0.0011416456636240088, - "sigma_level": 44.864361894597046, - "cpk": 14.680822923886454, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.20982575416564941, - "mean_abs_diff": 0.031462863087654114, - "cosine_similarity": 0.9999397993087769, - "kl_divergence": 0.0005747938176134488, - "sigma_level": 99.36599836682235, - "cpk": 32.86147122209217, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.24659061431884766, - "mean_abs_diff": 0.04355722293257713, - "cosine_similarity": 0.9999262094497681, - "kl_divergence": 0.001248096566017884, - "sigma_level": 74.74092104068832, - "cpk": 24.642348100233157, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.21674823760986328, - "mean_abs_diff": 0.030111797153949738, - "cosine_similarity": 0.9999454617500305, - "kl_divergence": 0.0008462897816879462, - "sigma_level": 103.61291944976223, - "cpk": 34.277642215504095, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1926441192626953, - "mean_abs_diff": 0.033761173486709595, - "cosine_similarity": 0.9999311566352844, - "kl_divergence": 0.0010025746520832025, - "sigma_level": 94.19188658464856, - "cpk": 31.132293142880787, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.2571578025817871, - "mean_abs_diff": 0.04913201928138733, - "cosine_similarity": 0.9998971223831177, - "kl_divergence": 0.00016394181358259913, - "sigma_level": 66.67619384362449, - "cpk": 21.952403277746956, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.20004218816757202, - "mean_abs_diff": 0.032239872962236404, - "cosine_similarity": 0.9999257326126099, - "kl_divergence": 0.00035964655689502595, - "sigma_level": 98.03535416893781, - "cpk": 32.41506410929474, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.22790932655334473, - "mean_abs_diff": 0.032669227570295334, - "cosine_similarity": 0.9999305009841919, - "kl_divergence": 0.00022142489203358796, - "sigma_level": 96.35532470982132, - "cpk": 31.856120400727534, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.20201587677001953, - "mean_abs_diff": 0.029387902468442917, - "cosine_similarity": 0.9999483227729797, - "kl_divergence": 0.0004107862516253817, - "sigma_level": 106.90799937791267, - "cpk": 35.374182971069686, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2490062713623047, - "mean_abs_diff": 0.03013196960091591, - "cosine_similarity": 0.999924898147583, - "kl_divergence": 0.00016009342785522766, - "sigma_level": 103.912201916083, - "cpk": 34.37647736291937, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19240188598632812, - "mean_abs_diff": 0.02893163077533245, - "cosine_similarity": 0.9999112486839294, - "kl_divergence": 6.817382872259671e-6, - "sigma_level": 108.02736989205229, - "cpk": 35.748672632405174, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.267974853515625, - "mean_abs_diff": 0.03601562976837158, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.00007162719322224682, - "sigma_level": 88.31999199829174, - "cpk": 29.174922321684246, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.24425232410430908, - "mean_abs_diff": 0.033856507390737534, - "cosine_similarity": 0.9999051690101624, - "kl_divergence": 0.0010536469720982588, - "sigma_level": 92.8341216476468, - "cpk": 30.682787288742585, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.20838356018066406, - "mean_abs_diff": 0.036430489271879196, - "cosine_similarity": 0.9999008774757385, - "kl_divergence": 0.0004580104239343563, - "sigma_level": 88.14959348258607, - "cpk": 29.115586759221365, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.21566104888916016, - "mean_abs_diff": 0.035551492124795914, - "cosine_similarity": 0.9999303221702576, - "kl_divergence": 0.00005223383602209498, - "sigma_level": 89.79387134077257, - "cpk": 29.66526493777199, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.19229507446289062, - "mean_abs_diff": 0.029516855254769325, - "cosine_similarity": 0.9999398589134216, - "kl_divergence": 0.00006387025116334416, - "sigma_level": 105.89702480105015, - "cpk": 35.03852933760311, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.21092748641967773, - "mean_abs_diff": 0.034741658717393875, - "cosine_similarity": 0.9999397993087769, - "kl_divergence": 0.000672524288997609, - "sigma_level": 92.15167487319876, - "cpk": 30.45043312117618, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2311105728149414, - "mean_abs_diff": 0.037915751338005066, - "cosine_similarity": 0.999920666217804, - "kl_divergence": 0.0001797125477679641, - "sigma_level": 84.79673733628577, - "cpk": 27.99765144483553, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.23068904876708984, - "mean_abs_diff": 0.0350475087761879, - "cosine_similarity": 0.9999470114707947, - "kl_divergence": 0.0010822221598805622, - "sigma_level": 91.749185936065, - "cpk": 30.315096945413142, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.17993736267089844, - "mean_abs_diff": 0.028637047857046127, - "cosine_similarity": 0.9999294877052307, - "kl_divergence": 0.00021486480617655506, - "sigma_level": 110.01873575701202, - "cpk": 36.41036093558357, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2231612205505371, - "mean_abs_diff": 0.04158709943294525, - "cosine_similarity": 0.9999403953552246, - "kl_divergence": 0.0008206746305046884, - "sigma_level": 78.4256339651499, - "cpk": 25.870086768566598, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.1945490837097168, - "mean_abs_diff": 0.03391721844673157, - "cosine_similarity": 0.9999347925186157, - "kl_divergence": 0.00013648398155169364, - "sigma_level": 95.6293235804782, - "cpk": 31.60615113867671, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18570899963378906, - "mean_abs_diff": 0.02739095687866211, - "cosine_similarity": 0.999912440776825, - "kl_divergence": 0.0006194080312011955, - "sigma_level": 114.51598314013269, - "cpk": 37.91060251703515, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.2039031982421875, - "mean_abs_diff": 0.029785331338644028, - "cosine_similarity": 0.9999380707740784, - "kl_divergence": 0.0008764585761620313, - "sigma_level": 105.47572482880632, - "cpk": 34.89677249191797, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.19327348470687866, - "mean_abs_diff": 0.0273088701069355, - "cosine_similarity": 0.9999405741691589, - "kl_divergence": 0.0003229069364669503, - "sigma_level": 115.07232534245732, - "cpk": 38.095567182012424, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.20456242561340332, - "mean_abs_diff": 0.03238324820995331, - "cosine_similarity": 0.9999169707298279, - "kl_divergence": 9.455850442395312e-6, - "sigma_level": 99.50280412389094, - "cpk": 32.89908270766946, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.2868785858154297, - "mean_abs_diff": 0.05693440139293671, - "cosine_similarity": 0.9998724460601807, - "kl_divergence": 0.0003243814681716116, - "sigma_level": 58.33631463373317, - "cpk": 19.168659615149256, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.25117525458335876, - "mean_abs_diff": 0.03555203229188919, - "cosine_similarity": 0.9999099969863892, - "kl_divergence": 0.0003880159170877742, - "sigma_level": 89.1645369985614, - "cpk": 29.45734729129845, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.28829193115234375, - "mean_abs_diff": 0.041850414127111435, - "cosine_similarity": 0.9998987317085266, - "kl_divergence": 0.0014878545190500573, - "sigma_level": 76.81169855905522, - "cpk": 25.336016070143128, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2536201477050781, - "mean_abs_diff": 0.03538894280791283, - "cosine_similarity": 0.9999024271965027, - "kl_divergence": 0.00042057384320396996, - "sigma_level": 90.20118474225366, - "cpk": 29.801051200080423, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.21267127990722656, - "mean_abs_diff": 0.034602489322423935, - "cosine_similarity": 0.9999219179153442, - "kl_divergence": 0.0017517055542956828, - "sigma_level": 92.15392100647352, - "cpk": 30.452244079854008, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.24674510955810547, - "mean_abs_diff": 0.028482820838689804, - "cosine_similarity": 0.9999575614929199, - "kl_divergence": 0.0009479499398340706, - "sigma_level": 109.40058760407271, - "cpk": 36.20719275665972, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2071094512939453, - "mean_abs_diff": 0.03381306678056717, - "cosine_similarity": 0.9999055862426758, - "kl_divergence": 0.0005057522906840319, - "sigma_level": 94.66239531405454, - "cpk": 31.287396280154628, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.2892899513244629, - "mean_abs_diff": 0.030646586790680885, - "cosine_similarity": 0.9998894929885864, - "kl_divergence": 0.0005098066796168379, - "sigma_level": 102.54242438017586, - "cpk": 33.91892685101747, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.7765593528747559, + "mean_abs_diff": 0.11612559109926224, + "cosine_similarity": 0.9986071586608887, + "kl_divergence": 0.011310374807126194, + "sigma_level": 27.6123564403692, + "cpk": 8.936910379016284, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.21668052673339844, + "mean_abs_diff": 0.03283289819955826, + "cosine_similarity": 0.9998907446861267, + "kl_divergence": 6.3402314768562945e-06, + "sigma_level": 96.19123042846297, + "cpk": 31.800557069791992, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.21230220794677734, + "mean_abs_diff": 0.036253008991479874, + "cosine_similarity": 0.9999125599861145, + "kl_divergence": 1.2138987899670097e-06, + "sigma_level": 87.78216091036128, + "cpk": 28.99552301438919, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.22992169857025146, + "mean_abs_diff": 0.031362131237983704, + "cosine_similarity": 0.9999329447746277, + "kl_divergence": 9.697837023952451e-05, + "sigma_level": 99.8193545508211, + "cpk": 33.01223920881423, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.20900297164916992, + "mean_abs_diff": 0.03212369233369827, + "cosine_similarity": 0.9999116063117981, + "kl_divergence": 1.2130307390144058e-06, + "sigma_level": 97.33557518850837, + "cpk": 32.184626890296194, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2239065170288086, + "mean_abs_diff": 0.033580731600522995, + "cosine_similarity": 0.9999029040336609, + "kl_divergence": 2.739006113745167e-05, + "sigma_level": 95.1876333412705, + "cpk": 31.462838583179984, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.26349353790283203, + "mean_abs_diff": 0.037349067628383636, + "cosine_similarity": 0.9999385476112366, + "kl_divergence": 6.901890175435653e-07, + "sigma_level": 85.80957537462608, + "cpk": 28.336116155389544, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.19062137603759766, + "mean_abs_diff": 0.03048071265220642, + "cosine_similarity": 0.9999557137489319, + "kl_divergence": 8.014137110110945e-07, + "sigma_level": 103.11592767396162, + "cpk": 34.11005531121256, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18280649185180664, + "mean_abs_diff": 0.02944883517920971, + "cosine_similarity": 0.9999102354049683, + "kl_divergence": 0.00036753254509921214, + "sigma_level": 106.09218564707784, + "cpk": 35.10370427494904, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2662510871887207, + "mean_abs_diff": 0.03750442713499069, + "cosine_similarity": 0.9999082684516907, + "kl_divergence": 0.0008122461170267515, + "sigma_level": 83.32380704426187, + "cpk": 27.514184710595494, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.21428704261779785, + "mean_abs_diff": 0.03618254512548447, + "cosine_similarity": 0.9999296069145203, + "kl_divergence": 0.0013414333957104756, + "sigma_level": 86.32583699113798, + "cpk": 28.514988289343737, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.2245645523071289, + "mean_abs_diff": 0.03425215184688568, + "cosine_similarity": 0.9999250173568726, + "kl_divergence": 0.0005234989769493335, + "sigma_level": 91.05669514978527, + "cpk": 30.09232440418292, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.24261903762817383, + "mean_abs_diff": 0.032688990235328674, + "cosine_similarity": 0.9999192953109741, + "kl_divergence": 0.00023210232981628998, + "sigma_level": 96.43691504783891, + "cpk": 31.882936234752634, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.19068384170532227, + "mean_abs_diff": 0.029661044478416443, + "cosine_similarity": 0.9999234080314636, + "kl_divergence": 0.00042034598196102077, + "sigma_level": 106.34280606113548, + "cpk": 35.184748795333576, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.22031402587890625, + "mean_abs_diff": 0.03211377561092377, + "cosine_similarity": 0.9999381899833679, + "kl_divergence": 0.00032284577336933807, + "sigma_level": 97.25797222535725, + "cpk": 32.15904735041753, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.26050472259521484, + "mean_abs_diff": 0.05659817159175873, + "cosine_similarity": 0.9997627139091492, + "kl_divergence": 0.0003138751302377445, + "sigma_level": 60.00271273002659, + "cpk": 19.71790059075344, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2220613956451416, + "mean_abs_diff": 0.034589797258377075, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 0.0010091621728843534, + "sigma_level": 90.97388370911588, + "cpk": 30.06239722026317, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.2452479600906372, + "mean_abs_diff": 0.04691462218761444, + "cosine_similarity": 0.9998961687088013, + "kl_divergence": 1.0286648444546095e-05, + "sigma_level": 71.21310678671547, + "cpk": 23.45929092892974, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.20011234283447266, + "mean_abs_diff": 0.028810735791921616, + "cosine_similarity": 0.9999183416366577, + "kl_divergence": 0.0005886006120770999, + "sigma_level": 109.42638674224135, + "cpk": 36.21274102098917, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 9904, + "gpu_argmax": 5942, + "max_abs_diff": 0.2104625701904297, + "mean_abs_diff": 0.03633837774395943, + "cosine_similarity": 0.9999009966850281, + "kl_divergence": 0.000780974228869925, + "sigma_level": 89.61552988240601, + "cpk": 29.60046971275267, + "verdict": "WarnArgmax" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.2549419403076172, + "mean_abs_diff": 0.03461437672376633, + "cosine_similarity": 0.9998999238014221, + "kl_divergence": 0.0005821044683649174, + "sigma_level": 90.29548641907701, + "cpk": 29.838035307745198, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.2721090316772461, + "mean_abs_diff": 0.041589464992284775, + "cosine_similarity": 0.9999185800552368, + "kl_divergence": 0.0012391113766812047, + "sigma_level": 75.04357270930153, + "cpk": 24.754439066426386, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.23730623722076416, + "mean_abs_diff": 0.040260475128889084, + "cosine_similarity": 0.9998979568481445, + "kl_divergence": 0.0009176755486284543, + "sigma_level": 79.46140806613252, + "cpk": 26.22053985178142, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19652128219604492, + "mean_abs_diff": 0.032911960035562515, + "cosine_similarity": 0.9999001026153564, + "kl_divergence": 0.00011332451760925956, + "sigma_level": 95.68482644088411, + "cpk": 31.632510881642027, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.1758362054824829, + "mean_abs_diff": 0.029195617884397507, + "cosine_similarity": 0.9998802542686462, + "kl_divergence": 0.0006681351605656909, + "sigma_level": 106.90541285517136, + "cpk": 35.37504015309934, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.18475735187530518, + "mean_abs_diff": 0.02708299830555916, + "cosine_similarity": 0.9998446702957153, + "kl_divergence": 6.509554974005071e-05, + "sigma_level": 115.87629249589668, + "cpk": 38.363907712522156, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.17609357833862305, + "mean_abs_diff": 0.027387676760554314, + "cosine_similarity": 0.9998769164085388, + "kl_divergence": 0.00018923296147374256, + "sigma_level": 114.58697406979748, + "cpk": 37.93413543936636, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.20337700843811035, + "mean_abs_diff": 0.0357307530939579, + "cosine_similarity": 0.9997212886810303, + "kl_divergence": 0.00068673561845549, + "sigma_level": 89.93727787207827, + "cpk": 29.7112987348936, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.2158222198486328, + "mean_abs_diff": 0.03296241909265518, + "cosine_similarity": 0.9998617768287659, + "kl_divergence": 0.0007174528587839378, + "sigma_level": 94.28973702732381, + "cpk": 31.170910856772032, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.18802005052566528, + "mean_abs_diff": 0.030701307579874992, + "cosine_similarity": 0.9998907446861267, + "kl_divergence": 0.000727208578105642, + "sigma_level": 102.51328926047806, + "cpk": 33.90882208477513, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.18789291381835938, + "mean_abs_diff": 0.028557559475302696, + "cosine_similarity": 0.9998390674591064, + "kl_divergence": 7.215432969221762e-05, + "sigma_level": 109.68622544018605, + "cpk": 36.30104423784289, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.1830120086669922, + "mean_abs_diff": 0.029256917536258698, + "cosine_similarity": 0.9998962879180908, + "kl_divergence": 0.0002161959497020309, + "sigma_level": 107.87296541749743, + "cpk": 35.69465260136486, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.18589067459106445, + "mean_abs_diff": 0.030194438993930817, + "cosine_similarity": 0.9999088644981384, + "kl_divergence": 0.0004802196691873215, + "sigma_level": 105.23769924870247, + "cpk": 34.81443364208194, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.19057416915893555, + "mean_abs_diff": 0.030395982787013054, + "cosine_similarity": 0.9998965263366699, + "kl_divergence": 0.0008087840754308433, + "sigma_level": 103.21133882875218, + "cpk": 34.14234543637878, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.20451754331588745, + "mean_abs_diff": 0.03417585417628288, + "cosine_similarity": 0.999851405620575, + "kl_divergence": 1.1874646119087019e-05, + "sigma_level": 93.7446562083357, + "cpk": 30.98123509441341, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.2111678123474121, + "mean_abs_diff": 0.031797345727682114, + "cosine_similarity": 0.9999288320541382, + "kl_divergence": 0.0003358035896646284, + "sigma_level": 98.93926386784285, + "cpk": 32.71758745767694, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.2316608428955078, + "mean_abs_diff": 0.03618849068880081, + "cosine_similarity": 0.999769926071167, + "kl_divergence": 0.0009839003068092891, + "sigma_level": 87.81894152240396, + "cpk": 29.008144261835998, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.21175193786621094, + "mean_abs_diff": 0.03807161748409271, + "cosine_similarity": 0.9998552203178406, + "kl_divergence": 0.00012716871719756028, + "sigma_level": 85.63607322735568, + "cpk": 28.273665757222588, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.22519683837890625, + "mean_abs_diff": 0.032994192093610764, + "cosine_similarity": 0.9999074339866638, + "kl_divergence": 0.001043569292129262, + "sigma_level": 94.46420366663784, + "cpk": 31.228337048733696, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.19043952226638794, + "mean_abs_diff": 0.030219942331314087, + "cosine_similarity": 0.999907910823822, + "kl_divergence": 0.0013804227424616085, + "sigma_level": 104.09966497830833, + "cpk": 34.437731170073306, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.19405746459960938, + "mean_abs_diff": 0.03200043737888336, + "cosine_similarity": 0.9999300837516785, + "kl_divergence": 0.0005307637452940784, + "sigma_level": 99.51118657903375, + "cpk": 32.90502873512619, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2977466583251953, + "mean_abs_diff": 0.04491940513253212, + "cosine_similarity": 0.9999076128005981, + "kl_divergence": 0.0019453823155301797, + "sigma_level": 71.66086038536118, + "cpk": 23.61870652680407, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.20647287368774414, + "mean_abs_diff": 0.029774188995361328, + "cosine_similarity": 0.9999259114265442, + "kl_divergence": 0.0004456308198784806, + "sigma_level": 105.13497563493686, + "cpk": 34.784132825430845, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.17754626274108887, + "mean_abs_diff": 0.028160495683550835, + "cosine_similarity": 0.9998858571052551, + "kl_divergence": 0.00016564224134654518, + "sigma_level": 111.07898880573893, + "cpk": 36.765659653179874, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.37882566452026367, + "mean_abs_diff": 0.0732780396938324, + "cosine_similarity": 0.9998091459274292, + "kl_divergence": 0.0011416456636240088, + "sigma_level": 44.864361894597046, + "cpk": 14.680822923886454, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.20982575416564941, + "mean_abs_diff": 0.031462863087654114, + "cosine_similarity": 0.9999397993087769, + "kl_divergence": 0.0005747938176134488, + "sigma_level": 99.36599836682235, + "cpk": 32.86147122209217, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.24659061431884766, + "mean_abs_diff": 0.04355722293257713, + "cosine_similarity": 0.9999262094497681, + "kl_divergence": 0.001248096566017884, + "sigma_level": 74.74092104068832, + "cpk": 24.642348100233157, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.21674823760986328, + "mean_abs_diff": 0.030111797153949738, + "cosine_similarity": 0.9999454617500305, + "kl_divergence": 0.0008462897816879462, + "sigma_level": 103.61291944976223, + "cpk": 34.277642215504095, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1926441192626953, + "mean_abs_diff": 0.033761173486709595, + "cosine_similarity": 0.9999311566352844, + "kl_divergence": 0.0010025746520832025, + "sigma_level": 94.19188658464856, + "cpk": 31.132293142880787, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.2571578025817871, + "mean_abs_diff": 0.04913201928138733, + "cosine_similarity": 0.9998971223831177, + "kl_divergence": 0.00016394181358259913, + "sigma_level": 66.67619384362449, + "cpk": 21.952403277746956, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.20004218816757202, + "mean_abs_diff": 0.032239872962236404, + "cosine_similarity": 0.9999257326126099, + "kl_divergence": 0.00035964655689502595, + "sigma_level": 98.03535416893781, + "cpk": 32.41506410929474, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.22790932655334473, + "mean_abs_diff": 0.032669227570295334, + "cosine_similarity": 0.9999305009841919, + "kl_divergence": 0.00022142489203358796, + "sigma_level": 96.35532470982132, + "cpk": 31.856120400727534, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.20201587677001953, + "mean_abs_diff": 0.029387902468442917, + "cosine_similarity": 0.9999483227729797, + "kl_divergence": 0.0004107862516253817, + "sigma_level": 106.90799937791267, + "cpk": 35.374182971069686, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2490062713623047, + "mean_abs_diff": 0.03013196960091591, + "cosine_similarity": 0.999924898147583, + "kl_divergence": 0.00016009342785522766, + "sigma_level": 103.912201916083, + "cpk": 34.37647736291937, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19240188598632812, + "mean_abs_diff": 0.02893163077533245, + "cosine_similarity": 0.9999112486839294, + "kl_divergence": 6.817382872259671e-06, + "sigma_level": 108.02736989205229, + "cpk": 35.748672632405174, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.267974853515625, + "mean_abs_diff": 0.03601562976837158, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 7.162719322224682e-05, + "sigma_level": 88.31999199829174, + "cpk": 29.174922321684246, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.24425232410430908, + "mean_abs_diff": 0.033856507390737534, + "cosine_similarity": 0.9999051690101624, + "kl_divergence": 0.0010536469720982588, + "sigma_level": 92.8341216476468, + "cpk": 30.682787288742585, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.20838356018066406, + "mean_abs_diff": 0.036430489271879196, + "cosine_similarity": 0.9999008774757385, + "kl_divergence": 0.0004580104239343563, + "sigma_level": 88.14959348258607, + "cpk": 29.115586759221365, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.21566104888916016, + "mean_abs_diff": 0.035551492124795914, + "cosine_similarity": 0.9999303221702576, + "kl_divergence": 5.223383602209498e-05, + "sigma_level": 89.79387134077257, + "cpk": 29.66526493777199, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.19229507446289062, + "mean_abs_diff": 0.029516855254769325, + "cosine_similarity": 0.9999398589134216, + "kl_divergence": 6.387025116334416e-05, + "sigma_level": 105.89702480105015, + "cpk": 35.03852933760311, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.21092748641967773, + "mean_abs_diff": 0.034741658717393875, + "cosine_similarity": 0.9999397993087769, + "kl_divergence": 0.000672524288997609, + "sigma_level": 92.15167487319876, + "cpk": 30.45043312117618, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2311105728149414, + "mean_abs_diff": 0.037915751338005066, + "cosine_similarity": 0.999920666217804, + "kl_divergence": 0.0001797125477679641, + "sigma_level": 84.79673733628577, + "cpk": 27.99765144483553, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.23068904876708984, + "mean_abs_diff": 0.0350475087761879, + "cosine_similarity": 0.9999470114707947, + "kl_divergence": 0.0010822221598805622, + "sigma_level": 91.749185936065, + "cpk": 30.315096945413142, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.17993736267089844, + "mean_abs_diff": 0.028637047857046127, + "cosine_similarity": 0.9999294877052307, + "kl_divergence": 0.00021486480617655506, + "sigma_level": 110.01873575701202, + "cpk": 36.41036093558357, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2231612205505371, + "mean_abs_diff": 0.04158709943294525, + "cosine_similarity": 0.9999403953552246, + "kl_divergence": 0.0008206746305046884, + "sigma_level": 78.4256339651499, + "cpk": 25.870086768566598, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.1945490837097168, + "mean_abs_diff": 0.03391721844673157, + "cosine_similarity": 0.9999347925186157, + "kl_divergence": 0.00013648398155169364, + "sigma_level": 95.6293235804782, + "cpk": 31.60615113867671, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18570899963378906, + "mean_abs_diff": 0.02739095687866211, + "cosine_similarity": 0.999912440776825, + "kl_divergence": 0.0006194080312011955, + "sigma_level": 114.51598314013269, + "cpk": 37.91060251703515, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.2039031982421875, + "mean_abs_diff": 0.029785331338644028, + "cosine_similarity": 0.9999380707740784, + "kl_divergence": 0.0008764585761620313, + "sigma_level": 105.47572482880632, + "cpk": 34.89677249191797, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.19327348470687866, + "mean_abs_diff": 0.0273088701069355, + "cosine_similarity": 0.9999405741691589, + "kl_divergence": 0.0003229069364669503, + "sigma_level": 115.07232534245732, + "cpk": 38.095567182012424, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.20456242561340332, + "mean_abs_diff": 0.03238324820995331, + "cosine_similarity": 0.9999169707298279, + "kl_divergence": 9.455850442395312e-06, + "sigma_level": 99.50280412389094, + "cpk": 32.89908270766946, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.2868785858154297, + "mean_abs_diff": 0.05693440139293671, + "cosine_similarity": 0.9998724460601807, + "kl_divergence": 0.0003243814681716116, + "sigma_level": 58.33631463373317, + "cpk": 19.168659615149256, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.25117525458335876, + "mean_abs_diff": 0.03555203229188919, + "cosine_similarity": 0.9999099969863892, + "kl_divergence": 0.0003880159170877742, + "sigma_level": 89.1645369985614, + "cpk": 29.45734729129845, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.28829193115234375, + "mean_abs_diff": 0.041850414127111435, + "cosine_similarity": 0.9998987317085266, + "kl_divergence": 0.0014878545190500573, + "sigma_level": 76.81169855905522, + "cpk": 25.336016070143128, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2536201477050781, + "mean_abs_diff": 0.03538894280791283, + "cosine_similarity": 0.9999024271965027, + "kl_divergence": 0.00042057384320396996, + "sigma_level": 90.20118474225366, + "cpk": 29.801051200080423, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.21267127990722656, + "mean_abs_diff": 0.034602489322423935, + "cosine_similarity": 0.9999219179153442, + "kl_divergence": 0.0017517055542956828, + "sigma_level": 92.15392100647352, + "cpk": 30.452244079854008, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.24674510955810547, + "mean_abs_diff": 0.028482820838689804, + "cosine_similarity": 0.9999575614929199, + "kl_divergence": 0.0009479499398340706, + "sigma_level": 109.40058760407271, + "cpk": 36.20719275665972, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2071094512939453, + "mean_abs_diff": 0.03381306678056717, + "cosine_similarity": 0.9999055862426758, + "kl_divergence": 0.0005057522906840319, + "sigma_level": 94.66239531405454, + "cpk": 31.287396280154628, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.2892899513244629, + "mean_abs_diff": 0.030646586790680885, + "cosine_similarity": 0.9998894929885864, + "kl_divergence": 0.0005098066796168379, + "sigma_level": 102.54242438017586, + "cpk": 33.91892685101747, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json b/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json index c29ba09d72..5d3189d2e7 100644 --- a/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1b/gx10/qwen2.5-coder-1.5b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-1.5b-instruct-q4_k_m", + "file": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "8cdb7c1a668127db", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md names the binary but carries no timestamp; the date is the git author date of the commit that ADDED this record \u2014 an upper bound on when it was measured, stated as such and not as a measurement", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9995829463005066, + "min_cosine_position": 22, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 16, - "gpu_argmax": 16, - "max_abs_diff": 0.17002153396606445, - "mean_abs_diff": 0.028188295662403107, - "cosine_similarity": 0.9999852776527405, - "kl_divergence": 0.0005291031695536818, - "sigma_level": 113.51094745441767, - "cpk": 37.57034247249217, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.31151044368743896, - "mean_abs_diff": 0.04776180535554886, - "cosine_similarity": 0.999913215637207, - "kl_divergence": 0.000018919660545105944, - "sigma_level": 65.37697481827193, - "cpk": 21.532114743923568, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.2960681915283203, - "mean_abs_diff": 0.05043606460094452, - "cosine_similarity": 0.9999374151229858, - "kl_divergence": 5.5291439027043675e-6, - "sigma_level": 61.84778951581121, - "cpk": 20.355983246316647, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.2933235168457031, - "mean_abs_diff": 0.04766261205077171, - "cosine_similarity": 0.9999246001243591, - "kl_divergence": 0.0001060628215343356, - "sigma_level": 66.19522509167993, - "cpk": 21.80215525279679, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.3401813507080078, - "mean_abs_diff": 0.05537569522857666, - "cosine_similarity": 0.9999253749847412, - "kl_divergence": 3.1806382760983147e-6, - "sigma_level": 58.14417981939996, - "cpk": 19.113078741383767, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.29981279373168945, - "mean_abs_diff": 0.04475972056388855, - "cosine_similarity": 0.9999157786369324, - "kl_divergence": 0.00006075290117685989, - "sigma_level": 70.78162520727061, - "cpk": 23.32986125531226, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.36730432510375977, - "mean_abs_diff": 0.05289534851908684, - "cosine_similarity": 0.9999330639839172, - "kl_divergence": 3.942939467082102e-6, - "sigma_level": 59.221911860577734, - "cpk": 19.47959031453992, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.37870264053344727, - "mean_abs_diff": 0.06961360573768616, - "cosine_similarity": 0.9999417066574097, - "kl_divergence": 5.816292943734862e-6, - "sigma_level": 47.307308057754874, - "cpk": 15.494666661614641, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2391514778137207, - "mean_abs_diff": 0.03515106812119484, - "cosine_similarity": 0.9999060034751892, - "kl_divergence": 0.00020459017940060442, - "sigma_level": 89.95740355697005, - "cpk": 29.72229295062028, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 498, - "gpu_argmax": 498, - "max_abs_diff": 0.2996964454650879, - "mean_abs_diff": 0.04814895987510681, - "cosine_similarity": 0.9999247193336487, - "kl_divergence": 0.0008225811496027273, - "sigma_level": 66.99980088661925, - "cpk": 22.064436068495585, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 7015, - "gpu_argmax": 7015, - "max_abs_diff": 0.2887105941772461, - "mean_abs_diff": 0.05602011829614639, - "cosine_similarity": 0.9999487400054932, - "kl_divergence": 0.0017507166523780418, - "sigma_level": 58.404129958133645, - "cpk": 19.19539279694137, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.23575425148010254, - "mean_abs_diff": 0.03749267756938934, - "cosine_similarity": 0.9999302625656128, - "kl_divergence": 0.0009205649545698543, - "sigma_level": 83.41319884576228, - "cpk": 27.5437842678078, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.23073053359985352, - "mean_abs_diff": 0.036286987364292145, - "cosine_similarity": 0.9999309778213501, - "kl_divergence": 0.0002103350091946032, - "sigma_level": 87.46254652146891, - "cpk": 28.889702813783522, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 35299, - "gpu_argmax": 35299, - "max_abs_diff": 0.22650158405303955, - "mean_abs_diff": 0.03817339614033699, - "cosine_similarity": 0.9999290108680725, - "kl_divergence": 0.0006199522697201657, - "sigma_level": 82.87208180962669, - "cpk": 27.360401535884456, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.2506117820739746, - "mean_abs_diff": 0.04409490525722504, - "cosine_similarity": 0.9999329447746277, - "kl_divergence": 0.001207147840417756, - "sigma_level": 72.75087140793437, - "cpk": 23.982961904135337, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2462446689605713, - "mean_abs_diff": 0.03615713492035866, - "cosine_similarity": 0.9999027252197266, - "kl_divergence": 0.0007957232607366745, - "sigma_level": 86.4713156254837, - "cpk": 28.563225623010272, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.30622434616088867, - "mean_abs_diff": 0.04285147041082382, - "cosine_similarity": 0.99992436170578, - "kl_divergence": 0.001562816857886753, - "sigma_level": 73.57721644186985, - "cpk": 24.262997821184154, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.26041460037231445, - "mean_abs_diff": 0.041633863002061844, - "cosine_similarity": 0.9999330043792725, - "kl_divergence": 0.0000386192979878687, - "sigma_level": 76.17994851863548, - "cpk": 25.129010711201076, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 8173, - "gpu_argmax": 8173, - "max_abs_diff": 0.2938199043273926, - "mean_abs_diff": 0.04622060805559158, - "cosine_similarity": 0.9999043345451355, - "kl_divergence": 0.0004776241426479791, - "sigma_level": 69.31179649604657, - "cpk": 22.836962717059464, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.33150339126586914, - "mean_abs_diff": 0.05491165816783905, - "cosine_similarity": 0.9998782873153687, - "kl_divergence": 0.004729332624206632, - "sigma_level": 59.30963249897647, - "cpk": 19.498478310838493, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 892, - "gpu_argmax": 892, - "max_abs_diff": 0.2719573974609375, - "mean_abs_diff": 0.0452549010515213, - "cosine_similarity": 0.9998787045478821, - "kl_divergence": 0.001985488312073216, - "sigma_level": 69.17673829662412, - "cpk": 22.798030561651316, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.5426011085510254, - "mean_abs_diff": 0.08933572471141815, - "cosine_similarity": 0.9997243881225586, - "kl_divergence": 0.0025612063383940826, - "sigma_level": 35.23386750744257, - "cpk": 11.482318911800565, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.6198992729187012, - "mean_abs_diff": 0.09157062321901321, - "cosine_similarity": 0.9995829463005066, - "kl_divergence": 0.010837422875741744, - "sigma_level": 34.05332167588209, - "cpk": 11.091250234582526, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.6332998275756836, - "mean_abs_diff": 0.0804496556520462, - "cosine_similarity": 0.9996832013130188, - "kl_divergence": 0.002525993250052904, - "sigma_level": 38.54398452241279, - "cpk": 12.589590650613772, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.35448646545410156, - "mean_abs_diff": 0.05539744719862938, - "cosine_similarity": 0.9998620748519897, - "kl_divergence": 0.002686264662680092, - "sigma_level": 56.62337561525491, - "cpk": 18.613059333347113, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3623623847961426, - "mean_abs_diff": 0.05665629357099533, - "cosine_similarity": 0.9997888207435608, - "kl_divergence": 0.00006107804999826922, - "sigma_level": 57.59732769966832, - "cpk": 18.927171640968005, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.26375818252563477, - "mean_abs_diff": 0.04152446612715721, - "cosine_similarity": 0.9999150633811951, - "kl_divergence": 0.00028931538420952634, - "sigma_level": 75.91556809492718, - "cpk": 25.042493245318916, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.298952579498291, - "mean_abs_diff": 0.05040597915649414, - "cosine_similarity": 0.9997300505638123, - "kl_divergence": 0.00029820851684663813, - "sigma_level": 63.07446362066332, - "cpk": 20.759877032006923, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.335299015045166, - "mean_abs_diff": 0.06050831079483032, - "cosine_similarity": 0.9998612999916077, - "kl_divergence": 0.002278695086270758, - "sigma_level": 52.97888535424999, - "cpk": 17.392489879701778, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.3436293601989746, - "mean_abs_diff": 0.05135822668671608, - "cosine_similarity": 0.9998526573181152, - "kl_divergence": 0.001024066884138274, - "sigma_level": 61.078798535758686, - "cpk": 20.098191280173584, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2161470353603363, - "mean_abs_diff": 0.03684602305293083, - "cosine_similarity": 0.9998604655265808, - "kl_divergence": 0.000037301007973426357, - "sigma_level": 84.97813534510328, - "cpk": 28.0651195872077, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3007354736328125, - "mean_abs_diff": 0.05120687186717987, - "cosine_similarity": 0.9999154806137085, - "kl_divergence": 0.0007108443470111474, - "sigma_level": 63.124975763940014, - "cpk": 20.772289209183093, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.28400611877441406, - "mean_abs_diff": 0.048652347177267075, - "cosine_similarity": 0.999927282333374, - "kl_divergence": 0.0015689714082204687, - "sigma_level": 65.72234032440733, - "cpk": 21.640984598238646, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 1075, - "gpu_argmax": 1075, - "max_abs_diff": 0.2669934034347534, - "mean_abs_diff": 0.04793161153793335, - "cosine_similarity": 0.9999324083328247, - "kl_divergence": 0.0025805480581009115, - "sigma_level": 67.08375790360458, - "cpk": 22.09329991583983, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.23727846145629883, - "mean_abs_diff": 0.0352792851626873, - "cosine_similarity": 0.9998976588249207, - "kl_divergence": 0.00003295202382643496, - "sigma_level": 89.6666882397245, - "cpk": 29.625281357907912, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 30128, - "gpu_argmax": 30128, - "max_abs_diff": 0.33716487884521484, - "mean_abs_diff": 0.05362644046545029, - "cosine_similarity": 0.9999046921730042, - "kl_divergence": 0.004216564679063551, - "sigma_level": 58.67804631412172, - "cpk": 19.297124208266144, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 1331, - "gpu_argmax": 1331, - "max_abs_diff": 0.3685007095336914, - "mean_abs_diff": 0.06397877633571625, - "cosine_similarity": 0.9996156692504883, - "kl_divergence": 0.005635637283317798, - "sigma_level": 50.13709882194314, - "cpk": 16.445057088010145, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.2645277976989746, - "mean_abs_diff": 0.034263480454683304, - "cosine_similarity": 0.9998966455459595, - "kl_divergence": 0.00022884919989930267, - "sigma_level": 91.22137614961048, - "cpk": 30.146661896640868, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 5819, - "gpu_argmax": 5819, - "max_abs_diff": 0.34772682189941406, - "mean_abs_diff": 0.0546145886182785, - "cosine_similarity": 0.9999197125434875, - "kl_divergence": 0.0028962852285730335, - "sigma_level": 58.42362283522166, - "cpk": 19.20864243451263, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.3002713918685913, - "mean_abs_diff": 0.04833799973130226, - "cosine_similarity": 0.9998859763145447, - "kl_divergence": 0.002065220995085909, - "sigma_level": 65.56426418444426, - "cpk": 21.59065094610386, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2973330020904541, - "mean_abs_diff": 0.043786294758319855, - "cosine_similarity": 0.9999096393585205, - "kl_divergence": 0.0006713194206523875, - "sigma_level": 71.27627178747183, - "cpk": 23.49868027534391, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.260500431060791, - "mean_abs_diff": 0.03889676183462143, - "cosine_similarity": 0.9999371767044067, - "kl_divergence": 0.0008542435718797348, - "sigma_level": 80.76395376693847, - "cpk": 26.65952989943824, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 24231, - "gpu_argmax": 24231, - "max_abs_diff": 0.27887630462646484, - "mean_abs_diff": 0.04313310235738754, - "cosine_similarity": 0.9999385476112366, - "kl_divergence": 0.0013764086829698126, - "sigma_level": 73.31553745426578, - "cpk": 24.174985269638448, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.234965980052948, - "mean_abs_diff": 0.037016309797763824, - "cosine_similarity": 0.9998894929885864, - "kl_divergence": 0.0002517016344552702, - "sigma_level": 85.95779722211104, - "cpk": 28.387445703078093, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2571530342102051, - "mean_abs_diff": 0.04071902483701706, - "cosine_similarity": 0.9998915791511536, - "kl_divergence": 0.002276788640036869, - "sigma_level": 76.79133459599869, - "cpk": 25.33653917694272, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.28516435623168945, - "mean_abs_diff": 0.04774056375026703, - "cosine_similarity": 0.9999310970306396, - "kl_divergence": 0.00042356634355521976, - "sigma_level": 68.00796327852372, - "cpk": 22.398759550639227, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.26332998275756836, - "mean_abs_diff": 0.0412241667509079, - "cosine_similarity": 0.9999069571495056, - "kl_divergence": 0.0015880368535440277, - "sigma_level": 76.05695528178228, - "cpk": 25.091036376668878, - "verdict": "Pass" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 5326, - "gpu_argmax": 5326, - "max_abs_diff": 0.27721452713012695, - "mean_abs_diff": 0.049495816230773926, - "cosine_similarity": 0.9999117851257324, - "kl_divergence": 0.0010624357608136137, - "sigma_level": 65.57931661159489, - "cpk": 21.589280386902693, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 504, - "gpu_argmax": 504, - "max_abs_diff": 0.27029943466186523, - "mean_abs_diff": 0.05219271406531334, - "cosine_similarity": 0.9999166131019592, - "kl_divergence": 0.0010408474118303833, - "sigma_level": 63.40393435111078, - "cpk": 20.85887616568664, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.3171241283416748, - "mean_abs_diff": 0.0380561500787735, - "cosine_similarity": 0.9999051690101624, - "kl_divergence": 0.0005325506114771998, - "sigma_level": 82.97525799995765, - "cpk": 27.395276094046604, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.24947690963745117, - "mean_abs_diff": 0.04606899619102478, - "cosine_similarity": 0.9999382495880127, - "kl_divergence": 0.0010556729962325755, - "sigma_level": 70.20693362744514, - "cpk": 23.13278096265952, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 1030, - "gpu_argmax": 1030, - "max_abs_diff": 0.2380300760269165, - "mean_abs_diff": 0.042963165789842606, - "cosine_similarity": 0.9999196529388428, - "kl_divergence": 0.00031761659666717314, - "sigma_level": 74.05351977421626, - "cpk": 24.419375454123664, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 29829, - "gpu_argmax": 29829, - "max_abs_diff": 0.25017356872558594, - "mean_abs_diff": 0.042943935841321945, - "cosine_similarity": 0.9999315142631531, - "kl_divergence": 0.0008225602750564301, - "sigma_level": 73.91521235873138, - "cpk": 24.373886608141206, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2465496063232422, - "mean_abs_diff": 0.04862823337316513, - "cosine_similarity": 0.9999127984046936, - "kl_divergence": 0.0005214034063651669, - "sigma_level": 67.83318248297935, - "cpk": 22.336176841974215, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.21641117334365845, - "mean_abs_diff": 0.035992056131362915, - "cosine_similarity": 0.9998767971992493, - "kl_divergence": 0.0010573586805331199, - "sigma_level": 87.34196263582032, - "cpk": 28.85201947678912, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.229217529296875, - "mean_abs_diff": 0.03840841352939606, - "cosine_similarity": 0.9999101161956787, - "kl_divergence": 0.001288439229203474, - "sigma_level": 82.85066673786027, - "cpk": 27.351708690182264, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 1467, - "gpu_argmax": 1467, - "max_abs_diff": 0.2576260566711426, - "mean_abs_diff": 0.043872181326150894, - "cosine_similarity": 0.9999276995658875, - "kl_divergence": 0.0013946707343927692, - "sigma_level": 73.02528098369798, - "cpk": 24.0747787971735, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.27596378326416016, - "mean_abs_diff": 0.05197622999548912, - "cosine_similarity": 0.999906599521637, - "kl_divergence": 0.0007940310091461293, - "sigma_level": 62.95701417717737, - "cpk": 20.71298237166727, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.21891295909881592, - "mean_abs_diff": 0.037558142095804214, - "cosine_similarity": 0.9999196529388428, - "kl_divergence": 0.00008131892636760107, - "sigma_level": 83.8883017292411, - "cpk": 27.70020984670349, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.21148943901062012, - "mean_abs_diff": 0.03717939928174019, - "cosine_similarity": 0.9999348521232605, - "kl_divergence": 0.00015387547222722972, - "sigma_level": 85.37753889289596, - "cpk": 28.194655830282713, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 12126, - "gpu_argmax": 12126, - "max_abs_diff": 0.2474832534790039, - "mean_abs_diff": 0.04211854934692383, - "cosine_similarity": 0.9999555349349976, - "kl_divergence": 0.0006737847648512362, - "sigma_level": 76.38969649351016, - "cpk": 25.19511356439018, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.21636962890625, - "mean_abs_diff": 0.035941481590270996, - "cosine_similarity": 0.9999298453330994, - "kl_divergence": 0.0004747428474169141, - "sigma_level": 88.5547413084973, - "cpk": 29.25301471912655, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2862367630004883, - "mean_abs_diff": 0.04804086312651634, - "cosine_similarity": 0.9999341368675232, - "kl_divergence": 0.0010998410243304767, - "sigma_level": 67.27443900835688, - "cpk": 22.15548615975949, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.2140594720840454, - "mean_abs_diff": 0.03911084309220314, - "cosine_similarity": 0.999923586845398, - "kl_divergence": 0.0005905188974057968, - "sigma_level": 82.12074070221703, - "cpk": 27.105929283720684, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.26285481452941895, - "mean_abs_diff": 0.041882410645484924, - "cosine_similarity": 0.9999344944953918, - "kl_divergence": 0.0014377984084730992, - "sigma_level": 75.78547694290033, - "cpk": 24.99731910877624, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 3501, - "gpu_argmax": 3501, - "max_abs_diff": 0.26805317401885986, - "mean_abs_diff": 0.03873350843787193, - "cosine_similarity": 0.9999223947525024, - "kl_divergence": 0.0013953240937074677, - "sigma_level": 81.43229589631191, - "cpk": 26.881252088752714, - "verdict": "Pass" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.21961069107055664, - "mean_abs_diff": 0.031359970569610596, - "cosine_similarity": 0.9999322295188904, - "kl_divergence": 0.0003661656672644013, - "sigma_level": 100.25205580134937, - "cpk": 33.155360140493684, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 1035, - "gpu_argmax": 1035, - "max_abs_diff": 0.24137163162231445, - "mean_abs_diff": 0.04070734232664108, - "cosine_similarity": 0.9999204277992249, - "kl_divergence": 0.0007682273957576004, - "sigma_level": 76.92578673738473, - "cpk": 25.38097521792283, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.22877216339111328, - "mean_abs_diff": 0.033897481858730316, - "cosine_similarity": 0.9999273419380188, - "kl_divergence": 0.0001278063653288519, - "sigma_level": 92.43480457949828, - "cpk": 30.550492600553692, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.2641177177429199, - "mean_abs_diff": 0.05057620257139206, - "cosine_similarity": 0.9998951554298401, - "kl_divergence": 0.0000393485153266477, - "sigma_level": 64.30636835231786, - "cpk": 21.164425124737836, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.2049417495727539, - "mean_abs_diff": 0.034037526696920395, - "cosine_similarity": 0.9999337792396545, - "kl_divergence": 0.00027996516407924826, - "sigma_level": 93.34949438746834, - "cpk": 30.851715970209657, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2751140594482422, - "mean_abs_diff": 0.04576941952109337, - "cosine_similarity": 0.9999250173568726, - "kl_divergence": 0.00035061780104783284, - "sigma_level": 69.71321462919332, - "cpk": 22.971843762520475, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.22286415100097656, - "mean_abs_diff": 0.039300888776779175, - "cosine_similarity": 0.9999452829360962, - "kl_divergence": 0.0003919376483464612, - "sigma_level": 79.54308945859913, - "cpk": 26.253853643551892, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2212667465209961, - "mean_abs_diff": 0.03471957892179489, - "cosine_similarity": 0.9999411702156067, - "kl_divergence": 0.00031067307671760625, - "sigma_level": 91.65536909076584, - "cpk": 30.286603378525857, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.24101543426513672, - "mean_abs_diff": 0.03968901187181473, - "cosine_similarity": 0.9999310970306396, - "kl_divergence": 0.0009727681612350478, - "sigma_level": 79.04490093748886, - "cpk": 26.086865811520088, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.2663121223449707, - "mean_abs_diff": 0.041511133313179016, - "cosine_similarity": 0.9999287128448486, - "kl_divergence": 0.0007963946022263226, - "sigma_level": 76.45829411292644, - "cpk": 25.221608834323817, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.2315669059753418, - "mean_abs_diff": 0.034654080867767334, - "cosine_similarity": 0.9999446272850037, - "kl_divergence": 0.0001058477485922799, - "sigma_level": 91.1050135078264, - "cpk": 30.105241127145526, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.3170022964477539, - "mean_abs_diff": 0.055937547236680984, - "cosine_similarity": 0.9998112916946411, - "kl_divergence": 0.0011604884668601218, - "sigma_level": 57.829405751003826, - "cpk": 19.00689899067911, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 16, + "gpu_argmax": 16, + "max_abs_diff": 0.17002153396606445, + "mean_abs_diff": 0.028188295662403107, + "cosine_similarity": 0.9999852776527405, + "kl_divergence": 0.0005291031695536818, + "sigma_level": 113.51094745441767, + "cpk": 37.57034247249217, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.31151044368743896, + "mean_abs_diff": 0.04776180535554886, + "cosine_similarity": 0.999913215637207, + "kl_divergence": 1.8919660545105944e-05, + "sigma_level": 65.37697481827193, + "cpk": 21.532114743923568, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.2960681915283203, + "mean_abs_diff": 0.05043606460094452, + "cosine_similarity": 0.9999374151229858, + "kl_divergence": 5.5291439027043675e-06, + "sigma_level": 61.84778951581121, + "cpk": 20.355983246316647, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.2933235168457031, + "mean_abs_diff": 0.04766261205077171, + "cosine_similarity": 0.9999246001243591, + "kl_divergence": 0.0001060628215343356, + "sigma_level": 66.19522509167993, + "cpk": 21.80215525279679, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.3401813507080078, + "mean_abs_diff": 0.05537569522857666, + "cosine_similarity": 0.9999253749847412, + "kl_divergence": 3.1806382760983147e-06, + "sigma_level": 58.14417981939996, + "cpk": 19.113078741383767, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.29981279373168945, + "mean_abs_diff": 0.04475972056388855, + "cosine_similarity": 0.9999157786369324, + "kl_divergence": 6.075290117685989e-05, + "sigma_level": 70.78162520727061, + "cpk": 23.32986125531226, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.36730432510375977, + "mean_abs_diff": 0.05289534851908684, + "cosine_similarity": 0.9999330639839172, + "kl_divergence": 3.942939467082102e-06, + "sigma_level": 59.221911860577734, + "cpk": 19.47959031453992, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.37870264053344727, + "mean_abs_diff": 0.06961360573768616, + "cosine_similarity": 0.9999417066574097, + "kl_divergence": 5.816292943734862e-06, + "sigma_level": 47.307308057754874, + "cpk": 15.494666661614641, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2391514778137207, + "mean_abs_diff": 0.03515106812119484, + "cosine_similarity": 0.9999060034751892, + "kl_divergence": 0.00020459017940060442, + "sigma_level": 89.95740355697005, + "cpk": 29.72229295062028, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 498, + "gpu_argmax": 498, + "max_abs_diff": 0.2996964454650879, + "mean_abs_diff": 0.04814895987510681, + "cosine_similarity": 0.9999247193336487, + "kl_divergence": 0.0008225811496027273, + "sigma_level": 66.99980088661925, + "cpk": 22.064436068495585, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 7015, + "gpu_argmax": 7015, + "max_abs_diff": 0.2887105941772461, + "mean_abs_diff": 0.05602011829614639, + "cosine_similarity": 0.9999487400054932, + "kl_divergence": 0.0017507166523780418, + "sigma_level": 58.404129958133645, + "cpk": 19.19539279694137, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.23575425148010254, + "mean_abs_diff": 0.03749267756938934, + "cosine_similarity": 0.9999302625656128, + "kl_divergence": 0.0009205649545698543, + "sigma_level": 83.41319884576228, + "cpk": 27.5437842678078, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.23073053359985352, + "mean_abs_diff": 0.036286987364292145, + "cosine_similarity": 0.9999309778213501, + "kl_divergence": 0.0002103350091946032, + "sigma_level": 87.46254652146891, + "cpk": 28.889702813783522, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 35299, + "gpu_argmax": 35299, + "max_abs_diff": 0.22650158405303955, + "mean_abs_diff": 0.03817339614033699, + "cosine_similarity": 0.9999290108680725, + "kl_divergence": 0.0006199522697201657, + "sigma_level": 82.87208180962669, + "cpk": 27.360401535884456, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.2506117820739746, + "mean_abs_diff": 0.04409490525722504, + "cosine_similarity": 0.9999329447746277, + "kl_divergence": 0.001207147840417756, + "sigma_level": 72.75087140793437, + "cpk": 23.982961904135337, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2462446689605713, + "mean_abs_diff": 0.03615713492035866, + "cosine_similarity": 0.9999027252197266, + "kl_divergence": 0.0007957232607366745, + "sigma_level": 86.4713156254837, + "cpk": 28.563225623010272, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.30622434616088867, + "mean_abs_diff": 0.04285147041082382, + "cosine_similarity": 0.99992436170578, + "kl_divergence": 0.001562816857886753, + "sigma_level": 73.57721644186985, + "cpk": 24.262997821184154, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.26041460037231445, + "mean_abs_diff": 0.041633863002061844, + "cosine_similarity": 0.9999330043792725, + "kl_divergence": 3.86192979878687e-05, + "sigma_level": 76.17994851863548, + "cpk": 25.129010711201076, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 8173, + "gpu_argmax": 8173, + "max_abs_diff": 0.2938199043273926, + "mean_abs_diff": 0.04622060805559158, + "cosine_similarity": 0.9999043345451355, + "kl_divergence": 0.0004776241426479791, + "sigma_level": 69.31179649604657, + "cpk": 22.836962717059464, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.33150339126586914, + "mean_abs_diff": 0.05491165816783905, + "cosine_similarity": 0.9998782873153687, + "kl_divergence": 0.004729332624206632, + "sigma_level": 59.30963249897647, + "cpk": 19.498478310838493, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 892, + "gpu_argmax": 892, + "max_abs_diff": 0.2719573974609375, + "mean_abs_diff": 0.0452549010515213, + "cosine_similarity": 0.9998787045478821, + "kl_divergence": 0.001985488312073216, + "sigma_level": 69.17673829662412, + "cpk": 22.798030561651316, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.5426011085510254, + "mean_abs_diff": 0.08933572471141815, + "cosine_similarity": 0.9997243881225586, + "kl_divergence": 0.0025612063383940826, + "sigma_level": 35.23386750744257, + "cpk": 11.482318911800565, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.6198992729187012, + "mean_abs_diff": 0.09157062321901321, + "cosine_similarity": 0.9995829463005066, + "kl_divergence": 0.010837422875741744, + "sigma_level": 34.05332167588209, + "cpk": 11.091250234582526, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.6332998275756836, + "mean_abs_diff": 0.0804496556520462, + "cosine_similarity": 0.9996832013130188, + "kl_divergence": 0.002525993250052904, + "sigma_level": 38.54398452241279, + "cpk": 12.589590650613772, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.35448646545410156, + "mean_abs_diff": 0.05539744719862938, + "cosine_similarity": 0.9998620748519897, + "kl_divergence": 0.002686264662680092, + "sigma_level": 56.62337561525491, + "cpk": 18.613059333347113, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3623623847961426, + "mean_abs_diff": 0.05665629357099533, + "cosine_similarity": 0.9997888207435608, + "kl_divergence": 6.107804999826922e-05, + "sigma_level": 57.59732769966832, + "cpk": 18.927171640968005, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.26375818252563477, + "mean_abs_diff": 0.04152446612715721, + "cosine_similarity": 0.9999150633811951, + "kl_divergence": 0.00028931538420952634, + "sigma_level": 75.91556809492718, + "cpk": 25.042493245318916, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.298952579498291, + "mean_abs_diff": 0.05040597915649414, + "cosine_similarity": 0.9997300505638123, + "kl_divergence": 0.00029820851684663813, + "sigma_level": 63.07446362066332, + "cpk": 20.759877032006923, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.335299015045166, + "mean_abs_diff": 0.06050831079483032, + "cosine_similarity": 0.9998612999916077, + "kl_divergence": 0.002278695086270758, + "sigma_level": 52.97888535424999, + "cpk": 17.392489879701778, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.3436293601989746, + "mean_abs_diff": 0.05135822668671608, + "cosine_similarity": 0.9998526573181152, + "kl_divergence": 0.001024066884138274, + "sigma_level": 61.078798535758686, + "cpk": 20.098191280173584, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2161470353603363, + "mean_abs_diff": 0.03684602305293083, + "cosine_similarity": 0.9998604655265808, + "kl_divergence": 3.7301007973426357e-05, + "sigma_level": 84.97813534510328, + "cpk": 28.0651195872077, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3007354736328125, + "mean_abs_diff": 0.05120687186717987, + "cosine_similarity": 0.9999154806137085, + "kl_divergence": 0.0007108443470111474, + "sigma_level": 63.124975763940014, + "cpk": 20.772289209183093, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.28400611877441406, + "mean_abs_diff": 0.048652347177267075, + "cosine_similarity": 0.999927282333374, + "kl_divergence": 0.0015689714082204687, + "sigma_level": 65.72234032440733, + "cpk": 21.640984598238646, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 1075, + "gpu_argmax": 1075, + "max_abs_diff": 0.2669934034347534, + "mean_abs_diff": 0.04793161153793335, + "cosine_similarity": 0.9999324083328247, + "kl_divergence": 0.0025805480581009115, + "sigma_level": 67.08375790360458, + "cpk": 22.09329991583983, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.23727846145629883, + "mean_abs_diff": 0.0352792851626873, + "cosine_similarity": 0.9998976588249207, + "kl_divergence": 3.295202382643496e-05, + "sigma_level": 89.6666882397245, + "cpk": 29.625281357907912, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 30128, + "gpu_argmax": 30128, + "max_abs_diff": 0.33716487884521484, + "mean_abs_diff": 0.05362644046545029, + "cosine_similarity": 0.9999046921730042, + "kl_divergence": 0.004216564679063551, + "sigma_level": 58.67804631412172, + "cpk": 19.297124208266144, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 1331, + "gpu_argmax": 1331, + "max_abs_diff": 0.3685007095336914, + "mean_abs_diff": 0.06397877633571625, + "cosine_similarity": 0.9996156692504883, + "kl_divergence": 0.005635637283317798, + "sigma_level": 50.13709882194314, + "cpk": 16.445057088010145, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.2645277976989746, + "mean_abs_diff": 0.034263480454683304, + "cosine_similarity": 0.9998966455459595, + "kl_divergence": 0.00022884919989930267, + "sigma_level": 91.22137614961048, + "cpk": 30.146661896640868, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 5819, + "gpu_argmax": 5819, + "max_abs_diff": 0.34772682189941406, + "mean_abs_diff": 0.0546145886182785, + "cosine_similarity": 0.9999197125434875, + "kl_divergence": 0.0028962852285730335, + "sigma_level": 58.42362283522166, + "cpk": 19.20864243451263, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.3002713918685913, + "mean_abs_diff": 0.04833799973130226, + "cosine_similarity": 0.9998859763145447, + "kl_divergence": 0.002065220995085909, + "sigma_level": 65.56426418444426, + "cpk": 21.59065094610386, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2973330020904541, + "mean_abs_diff": 0.043786294758319855, + "cosine_similarity": 0.9999096393585205, + "kl_divergence": 0.0006713194206523875, + "sigma_level": 71.27627178747183, + "cpk": 23.49868027534391, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.260500431060791, + "mean_abs_diff": 0.03889676183462143, + "cosine_similarity": 0.9999371767044067, + "kl_divergence": 0.0008542435718797348, + "sigma_level": 80.76395376693847, + "cpk": 26.65952989943824, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 24231, + "gpu_argmax": 24231, + "max_abs_diff": 0.27887630462646484, + "mean_abs_diff": 0.04313310235738754, + "cosine_similarity": 0.9999385476112366, + "kl_divergence": 0.0013764086829698126, + "sigma_level": 73.31553745426578, + "cpk": 24.174985269638448, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.234965980052948, + "mean_abs_diff": 0.037016309797763824, + "cosine_similarity": 0.9998894929885864, + "kl_divergence": 0.0002517016344552702, + "sigma_level": 85.95779722211104, + "cpk": 28.387445703078093, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2571530342102051, + "mean_abs_diff": 0.04071902483701706, + "cosine_similarity": 0.9998915791511536, + "kl_divergence": 0.002276788640036869, + "sigma_level": 76.79133459599869, + "cpk": 25.33653917694272, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.28516435623168945, + "mean_abs_diff": 0.04774056375026703, + "cosine_similarity": 0.9999310970306396, + "kl_divergence": 0.00042356634355521976, + "sigma_level": 68.00796327852372, + "cpk": 22.398759550639227, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.26332998275756836, + "mean_abs_diff": 0.0412241667509079, + "cosine_similarity": 0.9999069571495056, + "kl_divergence": 0.0015880368535440277, + "sigma_level": 76.05695528178228, + "cpk": 25.091036376668878, + "verdict": "Pass" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 5326, + "gpu_argmax": 5326, + "max_abs_diff": 0.27721452713012695, + "mean_abs_diff": 0.049495816230773926, + "cosine_similarity": 0.9999117851257324, + "kl_divergence": 0.0010624357608136137, + "sigma_level": 65.57931661159489, + "cpk": 21.589280386902693, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 504, + "gpu_argmax": 504, + "max_abs_diff": 0.27029943466186523, + "mean_abs_diff": 0.05219271406531334, + "cosine_similarity": 0.9999166131019592, + "kl_divergence": 0.0010408474118303833, + "sigma_level": 63.40393435111078, + "cpk": 20.85887616568664, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.3171241283416748, + "mean_abs_diff": 0.0380561500787735, + "cosine_similarity": 0.9999051690101624, + "kl_divergence": 0.0005325506114771998, + "sigma_level": 82.97525799995765, + "cpk": 27.395276094046604, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.24947690963745117, + "mean_abs_diff": 0.04606899619102478, + "cosine_similarity": 0.9999382495880127, + "kl_divergence": 0.0010556729962325755, + "sigma_level": 70.20693362744514, + "cpk": 23.13278096265952, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 1030, + "gpu_argmax": 1030, + "max_abs_diff": 0.2380300760269165, + "mean_abs_diff": 0.042963165789842606, + "cosine_similarity": 0.9999196529388428, + "kl_divergence": 0.00031761659666717314, + "sigma_level": 74.05351977421626, + "cpk": 24.419375454123664, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 29829, + "gpu_argmax": 29829, + "max_abs_diff": 0.25017356872558594, + "mean_abs_diff": 0.042943935841321945, + "cosine_similarity": 0.9999315142631531, + "kl_divergence": 0.0008225602750564301, + "sigma_level": 73.91521235873138, + "cpk": 24.373886608141206, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2465496063232422, + "mean_abs_diff": 0.04862823337316513, + "cosine_similarity": 0.9999127984046936, + "kl_divergence": 0.0005214034063651669, + "sigma_level": 67.83318248297935, + "cpk": 22.336176841974215, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.21641117334365845, + "mean_abs_diff": 0.035992056131362915, + "cosine_similarity": 0.9998767971992493, + "kl_divergence": 0.0010573586805331199, + "sigma_level": 87.34196263582032, + "cpk": 28.85201947678912, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.229217529296875, + "mean_abs_diff": 0.03840841352939606, + "cosine_similarity": 0.9999101161956787, + "kl_divergence": 0.001288439229203474, + "sigma_level": 82.85066673786027, + "cpk": 27.351708690182264, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 1467, + "gpu_argmax": 1467, + "max_abs_diff": 0.2576260566711426, + "mean_abs_diff": 0.043872181326150894, + "cosine_similarity": 0.9999276995658875, + "kl_divergence": 0.0013946707343927692, + "sigma_level": 73.02528098369798, + "cpk": 24.0747787971735, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.27596378326416016, + "mean_abs_diff": 0.05197622999548912, + "cosine_similarity": 0.999906599521637, + "kl_divergence": 0.0007940310091461293, + "sigma_level": 62.95701417717737, + "cpk": 20.71298237166727, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.21891295909881592, + "mean_abs_diff": 0.037558142095804214, + "cosine_similarity": 0.9999196529388428, + "kl_divergence": 8.131892636760107e-05, + "sigma_level": 83.8883017292411, + "cpk": 27.70020984670349, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.21148943901062012, + "mean_abs_diff": 0.03717939928174019, + "cosine_similarity": 0.9999348521232605, + "kl_divergence": 0.00015387547222722972, + "sigma_level": 85.37753889289596, + "cpk": 28.194655830282713, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 12126, + "gpu_argmax": 12126, + "max_abs_diff": 0.2474832534790039, + "mean_abs_diff": 0.04211854934692383, + "cosine_similarity": 0.9999555349349976, + "kl_divergence": 0.0006737847648512362, + "sigma_level": 76.38969649351016, + "cpk": 25.19511356439018, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.21636962890625, + "mean_abs_diff": 0.035941481590270996, + "cosine_similarity": 0.9999298453330994, + "kl_divergence": 0.0004747428474169141, + "sigma_level": 88.5547413084973, + "cpk": 29.25301471912655, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2862367630004883, + "mean_abs_diff": 0.04804086312651634, + "cosine_similarity": 0.9999341368675232, + "kl_divergence": 0.0010998410243304767, + "sigma_level": 67.27443900835688, + "cpk": 22.15548615975949, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.2140594720840454, + "mean_abs_diff": 0.03911084309220314, + "cosine_similarity": 0.999923586845398, + "kl_divergence": 0.0005905188974057968, + "sigma_level": 82.12074070221703, + "cpk": 27.105929283720684, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.26285481452941895, + "mean_abs_diff": 0.041882410645484924, + "cosine_similarity": 0.9999344944953918, + "kl_divergence": 0.0014377984084730992, + "sigma_level": 75.78547694290033, + "cpk": 24.99731910877624, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 3501, + "gpu_argmax": 3501, + "max_abs_diff": 0.26805317401885986, + "mean_abs_diff": 0.03873350843787193, + "cosine_similarity": 0.9999223947525024, + "kl_divergence": 0.0013953240937074677, + "sigma_level": 81.43229589631191, + "cpk": 26.881252088752714, + "verdict": "Pass" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.21961069107055664, + "mean_abs_diff": 0.031359970569610596, + "cosine_similarity": 0.9999322295188904, + "kl_divergence": 0.0003661656672644013, + "sigma_level": 100.25205580134937, + "cpk": 33.155360140493684, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 1035, + "gpu_argmax": 1035, + "max_abs_diff": 0.24137163162231445, + "mean_abs_diff": 0.04070734232664108, + "cosine_similarity": 0.9999204277992249, + "kl_divergence": 0.0007682273957576004, + "sigma_level": 76.92578673738473, + "cpk": 25.38097521792283, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.22877216339111328, + "mean_abs_diff": 0.033897481858730316, + "cosine_similarity": 0.9999273419380188, + "kl_divergence": 0.0001278063653288519, + "sigma_level": 92.43480457949828, + "cpk": 30.550492600553692, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.2641177177429199, + "mean_abs_diff": 0.05057620257139206, + "cosine_similarity": 0.9998951554298401, + "kl_divergence": 3.93485153266477e-05, + "sigma_level": 64.30636835231786, + "cpk": 21.164425124737836, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.2049417495727539, + "mean_abs_diff": 0.034037526696920395, + "cosine_similarity": 0.9999337792396545, + "kl_divergence": 0.00027996516407924826, + "sigma_level": 93.34949438746834, + "cpk": 30.851715970209657, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2751140594482422, + "mean_abs_diff": 0.04576941952109337, + "cosine_similarity": 0.9999250173568726, + "kl_divergence": 0.00035061780104783284, + "sigma_level": 69.71321462919332, + "cpk": 22.971843762520475, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.22286415100097656, + "mean_abs_diff": 0.039300888776779175, + "cosine_similarity": 0.9999452829360962, + "kl_divergence": 0.0003919376483464612, + "sigma_level": 79.54308945859913, + "cpk": 26.253853643551892, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2212667465209961, + "mean_abs_diff": 0.03471957892179489, + "cosine_similarity": 0.9999411702156067, + "kl_divergence": 0.00031067307671760625, + "sigma_level": 91.65536909076584, + "cpk": 30.286603378525857, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.24101543426513672, + "mean_abs_diff": 0.03968901187181473, + "cosine_similarity": 0.9999310970306396, + "kl_divergence": 0.0009727681612350478, + "sigma_level": 79.04490093748886, + "cpk": 26.086865811520088, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.2663121223449707, + "mean_abs_diff": 0.041511133313179016, + "cosine_similarity": 0.9999287128448486, + "kl_divergence": 0.0007963946022263226, + "sigma_level": 76.45829411292644, + "cpk": 25.221608834323817, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.2315669059753418, + "mean_abs_diff": 0.034654080867767334, + "cosine_similarity": 0.9999446272850037, + "kl_divergence": 0.0001058477485922799, + "sigma_level": 91.1050135078264, + "cpk": 30.105241127145526, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.3170022964477539, + "mean_abs_diff": 0.055937547236680984, + "cosine_similarity": 0.9998112916946411, + "kl_divergence": 0.0011604884668601218, + "sigma_level": 57.829405751003826, + "cpk": 19.00689899067911, + "verdict": "Pass" + } ] + } } diff --git a/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json b/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json index 9ddff2dde2..98aa405f04 100644 --- a/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json +++ b/evidence/parity/l0-1b/gx10/qwen2.5-coder-7b-instruct-q4_k_m.json @@ -1,1023 +1,1067 @@ { + "schema": "apr-parity-receipt/v2", + "cell": { + "model": "qwen2.5-coder-7b-instruct-q4_k_m", + "file": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", + "quant": "Q4_K_M" + }, + "host": "gx10-a5b5", + "backend": "cuda", + "apr_version": "0.65.2", + "generated_at": "2026-09-09", + "comparator": { + "kind": "self", + "reason": "No oracle arm exists for this cell. `apr parity` compares apr's own CPU and CUDA forwards on one binary; it is self-consistency, not parity against the pinned llama.cpp comparator (scripts/llama_pin.toml). The comparator ruling (#3577) schedules the dense oracle re-measurement as item (d); until it is taken this receipt states its basis rather than implying one." + }, + "partially_receipted": true, + "threshold_source": "evidence/parity/thresholds.yaml", + "unmeasured": [ + "ORACLE ARM: not measured. This is apr-CPU vs apr-CUDA on one binary, not apr vs llama.cpp.", + "The exact minute of the run is not recorded; see provenance.generated_at_basis.", + "model_sha256: NOT RECORDED at measurement time and deliberately not back-filled. Hashing the file on the host today would attach a claim about a different world to a receipt about 2026-09-09. Absent means no measurement, never a match (ONT-4c1)." + ], + "provenance": { + "record": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md", + "command": "apr parity --prompt \"\" --json", + "binary_sha256_prefix": "8cdb7c1a668127db", + "gpu": "NVIDIA GB10", + "arch": "aarch64", + "sm": "121", + "generated_at_basis": "evidence/parity/l0-1b/gx10/n5/DETERMINISM.md names the binary but carries no timestamp; the date is the git author date of the commit that ADDED this record \u2014 an upper bound on when it was measured, stated as such and not as a measurement", + "relabelled_by": "PMAT-3577 / #3577 \u2014 a relabel, not a re-measurement. `raw` below is the original `apr parity --json` document, key for key and value for value; every envelope field is quoted from the file named in `record`." + }, + "result": { + "positions": 78, + "parity": true, + "passed": 78, + "failed": 0, + "min_cosine": 0.9997819066047668, + "min_cosine_position": 24, + "threshold": 0.98, + "verdict": "PASS", + "judged_by": "scripts/check_model_parity.sh --judge (min cosine over >= 64 positions >= threshold)" + }, + "raw": { "model": "./qwen2.5-coder-7b-instruct-q4_k_m.gguf", "tokens": 78, "passed": 78, "failed": 0, "parity": true, "metrics": [ - { - "position": 0, - "token_id": 785, - "cpu_argmax": 914, - "gpu_argmax": 914, - "max_abs_diff": 0.22080087661743164, - "mean_abs_diff": 0.029157055541872978, - "cosine_similarity": 0.9999111890792847, - "kl_divergence": 0.00032579414447661754, - "sigma_level": 108.38003583659224, - "cpk": 35.863341718487604, - "verdict": "Pass" - }, - { - "position": 1, - "token_id": 3974, - "cpu_argmax": 13876, - "gpu_argmax": 13876, - "max_abs_diff": 0.1777787208557129, - "mean_abs_diff": 0.03252752497792244, - "cosine_similarity": 0.9999105930328369, - "kl_divergence": 3.7201543263529397e-6, - "sigma_level": 99.81459433877987, - "cpk": 33.000971303716966, - "verdict": "Pass" - }, - { - "position": 2, - "token_id": 13876, - "cpu_argmax": 38835, - "gpu_argmax": 38835, - "max_abs_diff": 0.17074143886566162, - "mean_abs_diff": 0.024813607335090637, - "cosine_similarity": 0.9999530911445618, - "kl_divergence": 6.065099658800252e-6, - "sigma_level": 126.29212369865193, - "cpk": 41.83622763563623, - "verdict": "Pass" - }, - { - "position": 3, - "token_id": 38835, - "cpu_argmax": 34208, - "gpu_argmax": 34208, - "max_abs_diff": 0.17793822288513184, - "mean_abs_diff": 0.029602359980344772, - "cosine_similarity": 0.9999439120292664, - "kl_divergence": 0.000021107606354716864, - "sigma_level": 106.68227318083109, - "cpk": 35.29758713909199, - "verdict": "Pass" - }, - { - "position": 4, - "token_id": 34208, - "cpu_argmax": 916, - "gpu_argmax": 916, - "max_abs_diff": 0.21715784072875977, - "mean_abs_diff": 0.033003006130456924, - "cosine_similarity": 0.9999298453330994, - "kl_divergence": 4.5147641864858906e-7, - "sigma_level": 97.43239515014821, - "cpk": 32.20950155551228, - "verdict": "Pass" - }, - { - "position": 5, - "token_id": 916, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.16768229007720947, - "mean_abs_diff": 0.026923540979623795, - "cosine_similarity": 0.9999296069145203, - "kl_divergence": 0.000024167411428152043, - "sigma_level": 115.93981600656502, - "cpk": 38.38647946990311, - "verdict": "Pass" - }, - { - "position": 6, - "token_id": 279, - "cpu_argmax": 15678, - "gpu_argmax": 15678, - "max_abs_diff": 0.1763458251953125, - "mean_abs_diff": 0.027678435668349266, - "cosine_similarity": 0.9999611973762512, - "kl_divergence": 1.2421132461965687e-6, - "sigma_level": 114.27627514935318, - "cpk": 37.8285093389394, - "verdict": "Pass" - }, - { - "position": 7, - "token_id": 15678, - "cpu_argmax": 5562, - "gpu_argmax": 5562, - "max_abs_diff": 0.2084965705871582, - "mean_abs_diff": 0.03252236545085907, - "cosine_similarity": 0.9999578595161438, - "kl_divergence": 1.444100421606142e-6, - "sigma_level": 98.27470173458114, - "cpk": 32.4918900978282, - "verdict": "Pass" - }, - { - "position": 8, - "token_id": 5562, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.2644195556640625, - "mean_abs_diff": 0.02975749969482422, - "cosine_similarity": 0.9999121427536011, - "kl_divergence": 0.0002420612186183089, - "sigma_level": 105.07405871764146, - "cpk": 34.764124466695144, - "verdict": "Pass" - }, - { - "position": 9, - "token_id": 1393, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.21603012084960938, - "mean_abs_diff": 0.03684907406568527, - "cosine_similarity": 0.9999308586120605, - "kl_divergence": 0.0006474370152407752, - "sigma_level": 86.82147331522204, - "cpk": 28.67388353001697, - "verdict": "Pass" - }, - { - "position": 10, - "token_id": 279, - "cpu_argmax": 8251, - "gpu_argmax": 8251, - "max_abs_diff": 0.2456355094909668, - "mean_abs_diff": 0.04065067693591118, - "cosine_similarity": 0.9999247193336487, - "kl_divergence": 0.001367731303832384, - "sigma_level": 78.56056947133877, - "cpk": 25.920728129656204, - "verdict": "Pass" - }, - { - "position": 11, - "token_id": 12801, - "cpu_argmax": 374, - "gpu_argmax": 374, - "max_abs_diff": 0.18509769439697266, - "mean_abs_diff": 0.03018302470445633, - "cosine_similarity": 0.9999426007270813, - "kl_divergence": 0.0005161167428984038, - "sigma_level": 104.15563778996108, - "cpk": 34.45656824761013, - "verdict": "Pass" - }, - { - "position": 12, - "token_id": 21926, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.17779541015625, - "mean_abs_diff": 0.031838010996580124, - "cosine_similarity": 0.999921977519989, - "kl_divergence": 0.00017258435469219227, - "sigma_level": 99.16622380548698, - "cpk": 32.792303324828296, - "verdict": "Pass" - }, - { - "position": 13, - "token_id": 35398, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.1694812774658203, - "mean_abs_diff": 0.026445960626006126, - "cosine_similarity": 0.9999368786811829, - "kl_divergence": 0.0005906903382915679, - "sigma_level": 118.52313191771943, - "cpk": 39.24650579924256, - "verdict": "Pass" - }, - { - "position": 14, - "token_id": 37402, - "cpu_argmax": 24258, - "gpu_argmax": 24258, - "max_abs_diff": 0.2208271026611328, - "mean_abs_diff": 0.0328725203871727, - "cosine_similarity": 0.9999487400054932, - "kl_divergence": 0.0007866923538006692, - "sigma_level": 96.84132305510252, - "cpk": 32.01515615449671, - "verdict": "Pass" - }, - { - "position": 15, - "token_id": 24258, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.18550491333007812, - "mean_abs_diff": 0.026621753349900246, - "cosine_similarity": 0.999907374382019, - "kl_divergence": 0.0002935938926612731, - "sigma_level": 117.17077206714823, - "cpk": 38.796983072900325, - "verdict": "Pass" - }, - { - "position": 16, - "token_id": 911, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.22534465789794922, - "mean_abs_diff": 0.035026710480451584, - "cosine_similarity": 0.9999264478683472, - "kl_divergence": 0.00047285014688066114, - "sigma_level": 90.85841261803856, - "cpk": 30.020931596555734, - "verdict": "Pass" - }, - { - "position": 17, - "token_id": 32168, - "cpu_argmax": 4802, - "gpu_argmax": 4802, - "max_abs_diff": 0.14307236671447754, - "mean_abs_diff": 0.02470676600933075, - "cosine_similarity": 0.999955952167511, - "kl_divergence": 0.00003484863188887505, - "sigma_level": 128.68804300700253, - "cpk": 42.631058888436456, - "verdict": "Pass" - }, - { - "position": 18, - "token_id": 4802, - "cpu_argmax": 7079, - "gpu_argmax": 7079, - "max_abs_diff": 0.19958877563476562, - "mean_abs_diff": 0.028368674218654633, - "cosine_similarity": 0.9999251365661621, - "kl_divergence": 0.00007519966075083051, - "sigma_level": 112.12267875571543, - "cpk": 37.1091619397265, - "verdict": "Pass" - }, - { - "position": 19, - "token_id": 5819, - "cpu_argmax": 5942, - "gpu_argmax": 5942, - "max_abs_diff": 0.1851489543914795, - "mean_abs_diff": 0.03547385707497597, - "cosine_similarity": 0.9999073147773743, - "kl_divergence": 0.0005868901702431452, - "sigma_level": 92.18467398700828, - "cpk": 30.455712499876228, - "verdict": "Pass" - }, - { - "position": 20, - "token_id": 11, - "cpu_argmax": 2670, - "gpu_argmax": 2670, - "max_abs_diff": 0.2207651138305664, - "mean_abs_diff": 0.03258340433239937, - "cosine_similarity": 0.9999098777770996, - "kl_divergence": 0.0009104736533284513, - "sigma_level": 96.75204615803823, - "cpk": 31.98797279934988, - "verdict": "Pass" - }, - { - "position": 21, - "token_id": 4237, - "cpu_argmax": 9471, - "gpu_argmax": 9471, - "max_abs_diff": 0.2389364242553711, - "mean_abs_diff": 0.038926344364881516, - "cosine_similarity": 0.9999305009841919, - "kl_divergence": 0.000779499423692475, - "sigma_level": 80.85147160964138, - "cpk": 26.688219517690097, - "verdict": "Pass" - }, - { - "position": 22, - "token_id": 23869, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.2731236219406128, - "mean_abs_diff": 0.04174664989113808, - "cosine_similarity": 0.9998652935028076, - "kl_divergence": 0.0016339679460959167, - "sigma_level": 74.70249389609015, - "cpk": 24.64094972714047, - "verdict": "Pass" - }, - { - "position": 23, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.24706506729125977, - "mean_abs_diff": 0.04125778004527092, - "cosine_similarity": 0.9998652935028076, - "kl_divergence": 0.00012628074726192903, - "sigma_level": 77.1027654306697, - "cpk": 25.43583106547151, - "verdict": "Pass" - }, - { - "position": 24, - "token_id": 15626, - "cpu_argmax": 14155, - "gpu_argmax": 14155, - "max_abs_diff": 0.23851919174194336, - "mean_abs_diff": 0.043194543570280075, - "cosine_similarity": 0.9997819066047668, - "kl_divergence": 0.0007928120095410873, - "sigma_level": 75.55819186163643, - "cpk": 24.914088819673893, - "verdict": "Pass" - }, - { - "position": 25, - "token_id": 49054, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1764129400253296, - "mean_abs_diff": 0.026423683390021324, - "cosine_similarity": 0.9998510479927063, - "kl_divergence": 0.00003542603986894582, - "sigma_level": 118.8965940005014, - "cpk": 39.37039083716537, - "verdict": "Pass" - }, - { - "position": 26, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19931602478027344, - "mean_abs_diff": 0.03260093927383423, - "cosine_similarity": 0.9998522400856018, - "kl_divergence": 0.00016412521494609694, - "sigma_level": 98.43831627071472, - "cpk": 32.54534029265824, - "verdict": "Pass" - }, - { - "position": 27, - "token_id": 10272, - "cpu_argmax": 2022, - "gpu_argmax": 2022, - "max_abs_diff": 0.16838788986206055, - "mean_abs_diff": 0.025229420512914658, - "cosine_similarity": 0.9998432993888855, - "kl_divergence": 0.00022452636227061513, - "sigma_level": 124.04995771005096, - "cpk": 41.08917685771064, - "verdict": "Pass" - }, - { - "position": 28, - "token_id": 1506, - "cpu_argmax": 29728, - "gpu_argmax": 29728, - "max_abs_diff": 0.18775105476379395, - "mean_abs_diff": 0.028898026794195175, - "cosine_similarity": 0.9998990893363953, - "kl_divergence": 0.000461885527735682, - "sigma_level": 108.11494274168895, - "cpk": 35.77795520454613, - "verdict": "Pass" - }, - { - "position": 29, - "token_id": 6529, - "cpu_argmax": 23783, - "gpu_argmax": 23783, - "max_abs_diff": 0.1990138292312622, - "mean_abs_diff": 0.02971040830016136, - "cosine_similarity": 0.9999052882194519, - "kl_divergence": 0.00019171666737919304, - "sigma_level": 107.355287869335, - "cpk": 35.51929850346339, - "verdict": "Pass" - }, - { - "position": 30, - "token_id": 63515, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1763872504234314, - "mean_abs_diff": 0.02600252814590931, - "cosine_similarity": 0.9998703598976135, - "kl_divergence": 5.419791774745481e-6, - "sigma_level": 121.23067932375403, - "cpk": 40.14753442864604, - "verdict": "Pass" - }, - { - "position": 31, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.19364482164382935, - "mean_abs_diff": 0.027832137420773506, - "cosine_similarity": 0.9999036192893982, - "kl_divergence": 0.00015532038116852645, - "sigma_level": 112.43699543023251, - "cpk": 37.21821831774474, - "verdict": "Pass" - }, - { - "position": 32, - "token_id": 323, - "cpu_argmax": 1008, - "gpu_argmax": 1008, - "max_abs_diff": 0.1895672082901001, - "mean_abs_diff": 0.028663579374551773, - "cosine_similarity": 0.9999141097068787, - "kl_divergence": 0.00048805987877351045, - "sigma_level": 110.13832391530234, - "cpk": 36.44969475595692, - "verdict": "Pass" - }, - { - "position": 33, - "token_id": 279, - "cpu_argmax": 990, - "gpu_argmax": 990, - "max_abs_diff": 0.1705029010772705, - "mean_abs_diff": 0.02920815348625183, - "cosine_similarity": 0.9999076724052429, - "kl_divergence": 0.0006465286747198685, - "sigma_level": 107.00006736446687, - "cpk": 35.40624958893724, - "verdict": "Pass" - }, - { - "position": 34, - "token_id": 27889, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.17264866828918457, - "mean_abs_diff": 0.027726825326681137, - "cosine_similarity": 0.9998999238014221, - "kl_divergence": 9.778256838183266e-6, - "sigma_level": 113.50747960523304, - "cpk": 37.5735596967205, - "verdict": "Pass" - }, - { - "position": 35, - "token_id": 315, - "cpu_argmax": 32168, - "gpu_argmax": 32168, - "max_abs_diff": 0.20781350135803223, - "mean_abs_diff": 0.0317465104162693, - "cosine_similarity": 0.9999275803565979, - "kl_divergence": 0.0007944463203620559, - "sigma_level": 98.32937454807063, - "cpk": 32.516323639913914, - "verdict": "Pass" - }, - { - "position": 36, - "token_id": 656, - "cpu_argmax": 59711, - "gpu_argmax": 59711, - "max_abs_diff": 0.16794657707214355, - "mean_abs_diff": 0.025144487619400024, - "cosine_similarity": 0.9998830556869507, - "kl_divergence": 0.0014841781022014834, - "sigma_level": 124.04210451408106, - "cpk": 41.08745357459047, - "verdict": "Pass" - }, - { - "position": 37, - "token_id": 38589, - "cpu_argmax": 291, - "gpu_argmax": 291, - "max_abs_diff": 0.14394879341125488, - "mean_abs_diff": 0.024181175976991653, - "cosine_similarity": 0.99992835521698, - "kl_divergence": 0.00004590317076718935, - "sigma_level": 131.6329585448339, - "cpk": 43.61239953703258, - "verdict": "Pass" - }, - { - "position": 38, - "token_id": 291, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.20485520362854004, - "mean_abs_diff": 0.032248660922050476, - "cosine_similarity": 0.9999155402183533, - "kl_divergence": 0.0010132197408340808, - "sigma_level": 97.76611926328172, - "cpk": 32.32597088527838, - "verdict": "Pass" - }, - { - "position": 39, - "token_id": 44378, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.17113685607910156, - "mean_abs_diff": 0.027429627254605293, - "cosine_similarity": 0.9999209046363831, - "kl_divergence": 0.00020635141816486075, - "sigma_level": 114.89910887906412, - "cpk": 38.03706631565145, - "verdict": "Pass" - }, - { - "position": 40, - "token_id": 3941, - "cpu_argmax": 5248, - "gpu_argmax": 5248, - "max_abs_diff": 0.20895957946777344, - "mean_abs_diff": 0.03152859956026077, - "cosine_similarity": 0.9999305605888367, - "kl_divergence": 0.0012345569716387572, - "sigma_level": 100.7526951257938, - "cpk": 33.31951576032809, - "verdict": "Pass" - }, - { - "position": 41, - "token_id": 3040, - "cpu_argmax": 2155, - "gpu_argmax": 2155, - "max_abs_diff": 0.2068023681640625, - "mean_abs_diff": 0.03304615244269371, - "cosine_similarity": 0.9999384880065918, - "kl_divergence": 0.0008537524363857488, - "sigma_level": 95.37599649244608, - "cpk": 31.52934802086009, - "verdict": "Pass" - }, - { - "position": 42, - "token_id": 97782, - "cpu_argmax": 821, - "gpu_argmax": 821, - "max_abs_diff": 0.19442176818847656, - "mean_abs_diff": 0.03274036571383476, - "cosine_similarity": 0.999926745891571, - "kl_divergence": 0.00022415574672985837, - "sigma_level": 96.70010563019095, - "cpk": 31.969535474822074, - "verdict": "Pass" - }, - { - "position": 43, - "token_id": 18432, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.1620619297027588, - "mean_abs_diff": 0.02630770020186901, - "cosine_similarity": 0.9999019503593445, - "kl_divergence": 0.00015874297246952666, - "sigma_level": 119.79814893017625, - "cpk": 39.670081827825925, - "verdict": "Pass" - }, - { - "position": 44, - "token_id": 26, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.2568702697753906, - "mean_abs_diff": 0.041561197489500046, - "cosine_similarity": 0.9999032020568848, - "kl_divergence": 0.0006645838048104513, - "sigma_level": 75.64519761529297, - "cpk": 24.953073788662536, - "verdict": "Pass" - }, - { - "position": 45, - "token_id": 1449, - "cpu_argmax": 13734, - "gpu_argmax": 13734, - "max_abs_diff": 0.23499298095703125, - "mean_abs_diff": 0.03136107325553894, - "cosine_similarity": 0.9999419450759888, - "kl_divergence": 0.0010564909150774027, - "sigma_level": 100.73015249089858, - "cpk": 33.31346702269047, - "verdict": "Pass" - }, - { - "position": 46, - "token_id": 14311, - "cpu_argmax": 304, - "gpu_argmax": 572, - "max_abs_diff": 0.19125032424926758, - "mean_abs_diff": 0.03027286008000374, - "cosine_similarity": 0.9999489188194275, - "kl_divergence": 0.0007324182535427381, - "sigma_level": 103.65502957340333, - "cpk": 34.290182007229085, - "verdict": "WarnArgmax" - }, - { - "position": 47, - "token_id": 572, - "cpu_argmax": 90326, - "gpu_argmax": 90326, - "max_abs_diff": 0.17596006393432617, - "mean_abs_diff": 0.028325680643320084, - "cosine_similarity": 0.9999551177024841, - "kl_divergence": 0.0004250593527600838, - "sigma_level": 111.41886548737655, - "cpk": 36.87662056233914, - "verdict": "Pass" - }, - { - "position": 48, - "token_id": 48826, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.1672215461730957, - "mean_abs_diff": 0.02670413628220558, - "cosine_similarity": 0.9999512434005737, - "kl_divergence": 0.0003977476078045197, - "sigma_level": 118.5229880052167, - "cpk": 39.24390816638344, - "verdict": "Pass" - }, - { - "position": 49, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.18988656997680664, - "mean_abs_diff": 0.029908394441008568, - "cosine_similarity": 0.999937117099762, - "kl_divergence": 0.00020613579089726636, - "sigma_level": 105.60082770042673, - "cpk": 34.9370799661288, - "verdict": "Pass" - }, - { - "position": 50, - "token_id": 1449, - "cpu_argmax": 11652, - "gpu_argmax": 11652, - "max_abs_diff": 0.18568384647369385, - "mean_abs_diff": 0.02834681235253811, - "cosine_similarity": 0.9999350309371948, - "kl_divergence": 0.0006665192974476394, - "sigma_level": 110.43670259979491, - "cpk": 36.55135682614585, - "verdict": "Pass" - }, - { - "position": 51, - "token_id": 1965, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.20452356338500977, - "mean_abs_diff": 0.0283601526170969, - "cosine_similarity": 0.9999455809593201, - "kl_divergence": 0.00007353835996759626, - "sigma_level": 110.17008996001424, - "cpk": 36.46299327291264, - "verdict": "Pass" - }, - { - "position": 52, - "token_id": 572, - "cpu_argmax": 17256, - "gpu_argmax": 17256, - "max_abs_diff": 0.17694008350372314, - "mean_abs_diff": 0.029460342600941658, - "cosine_similarity": 0.999952495098114, - "kl_divergence": 0.000354042526327484, - "sigma_level": 108.24661766438967, - "cpk": 35.81645735131439, - "verdict": "Pass" - }, - { - "position": 53, - "token_id": 21870, - "cpu_argmax": 11, - "gpu_argmax": 11, - "max_abs_diff": 0.17319059371948242, - "mean_abs_diff": 0.025698617100715637, - "cosine_similarity": 0.9999451041221619, - "kl_divergence": 0.0012082822780696116, - "sigma_level": 121.84608568951509, - "cpk": 40.35442223805871, - "verdict": "Pass" - }, - { - "position": 54, - "token_id": 11, - "cpu_argmax": 323, - "gpu_argmax": 323, - "max_abs_diff": 0.18508267402648926, - "mean_abs_diff": 0.026767021045088768, - "cosine_similarity": 0.9999246597290039, - "kl_divergence": 8.510791162766133e-6, - "sigma_level": 117.67908059360465, - "cpk": 38.96386699563357, - "verdict": "Pass" - }, - { - "position": 55, - "token_id": 323, - "cpu_argmax": 1449, - "gpu_argmax": 1449, - "max_abs_diff": 0.23066043853759766, - "mean_abs_diff": 0.036110661923885345, - "cosine_similarity": 0.9999273419380188, - "kl_divergence": 0.0004994907989914782, - "sigma_level": 88.93142368056236, - "cpk": 29.3761935122759, - "verdict": "Pass" - }, - { - "position": 56, - "token_id": 279, - "cpu_argmax": 2197, - "gpu_argmax": 2197, - "max_abs_diff": 0.18856239318847656, - "mean_abs_diff": 0.030129026621580124, - "cosine_similarity": 0.9999247789382935, - "kl_divergence": 0.0008435059720457446, - "sigma_level": 104.45399344155254, - "cpk": 34.55573971808994, - "verdict": "Pass" - }, - { - "position": 57, - "token_id": 1895, - "cpu_argmax": 572, - "gpu_argmax": 572, - "max_abs_diff": 0.16795825958251953, - "mean_abs_diff": 0.0249911118298769, - "cosine_similarity": 0.9999393820762634, - "kl_divergence": 0.0001998253593928442, - "sigma_level": 124.82156616889559, - "cpk": 41.34723624638959, - "verdict": "Pass" - }, - { - "position": 58, - "token_id": 9482, - "cpu_argmax": 448, - "gpu_argmax": 448, - "max_abs_diff": 0.17892026901245117, - "mean_abs_diff": 0.028010983020067215, - "cosine_similarity": 0.9999496936798096, - "kl_divergence": 0.000045661537655344266, - "sigma_level": 112.21705530380771, - "cpk": 37.143742598712834, - "verdict": "Pass" - }, - { - "position": 59, - "token_id": 448, - "cpu_argmax": 264, - "gpu_argmax": 264, - "max_abs_diff": 0.23161697387695312, - "mean_abs_diff": 0.03567603975534439, - "cosine_similarity": 0.9999344348907471, - "kl_divergence": 0.0002818625074985845, - "sigma_level": 90.15571250675596, - "cpk": 29.78387093695512, - "verdict": "Pass" - }, - { - "position": 60, - "token_id": 264, - "cpu_argmax": 11682, - "gpu_argmax": 11682, - "max_abs_diff": 0.17027735710144043, - "mean_abs_diff": 0.026973217725753784, - "cosine_similarity": 0.9999549388885498, - "kl_divergence": 0.0005629856415008736, - "sigma_level": 116.05363658933409, - "cpk": 38.423683862478896, - "verdict": "Pass" - }, - { - "position": 61, - "token_id": 52573, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.1832141876220703, - "mean_abs_diff": 0.02868635207414627, - "cosine_similarity": 0.999939501285553, - "kl_divergence": 0.0006119540337820587, - "sigma_level": 109.6258088123588, - "cpk": 36.27987255845257, - "verdict": "Pass" - }, - { - "position": 62, - "token_id": 315, - "cpu_argmax": 1917, - "gpu_argmax": 1917, - "max_abs_diff": 0.2024369239807129, - "mean_abs_diff": 0.0438840389251709, - "cosine_similarity": 0.9999357461929321, - "kl_divergence": 0.00047113813301219134, - "sigma_level": 75.99682576801438, - "cpk": 25.054354617655378, - "verdict": "Pass" - }, - { - "position": 63, - "token_id": 52374, - "cpu_argmax": 69715, - "gpu_argmax": 69715, - "max_abs_diff": 0.18555545806884766, - "mean_abs_diff": 0.025885246694087982, - "cosine_similarity": 0.9999472498893738, - "kl_divergence": 0.000672849308967897, - "sigma_level": 121.69151806777218, - "cpk": 40.301338108777195, - "verdict": "Pass" - }, - { - "position": 64, - "token_id": 41017, - "cpu_argmax": 3589, - "gpu_argmax": 3589, - "max_abs_diff": 0.16499638557434082, - "mean_abs_diff": 0.02714230865240097, - "cosine_similarity": 0.9999595284461975, - "kl_divergence": 0.000296325939274267, - "sigma_level": 114.99287429648993, - "cpk": 38.070860424914805, - "verdict": "Pass" - }, - { - "position": 65, - "token_id": 22901, - "cpu_argmax": 9079, - "gpu_argmax": 3589, - "max_abs_diff": 0.17669677734375, - "mean_abs_diff": 0.02647567354142666, - "cosine_similarity": 0.9999456405639648, - "kl_divergence": 0.00043440203458147177, - "sigma_level": 119.37552964782644, - "cpk": 39.52846425329292, - "verdict": "WarnArgmax" - }, - { - "position": 66, - "token_id": 7354, - "cpu_argmax": 13, - "gpu_argmax": 13, - "max_abs_diff": 0.15739870071411133, - "mean_abs_diff": 0.0278735663741827, - "cosine_similarity": 0.999921977519989, - "kl_divergence": 0.0001710410126672348, - "sigma_level": 114.46856639915022, - "cpk": 37.89030153444305, - "verdict": "Pass" - }, - { - "position": 67, - "token_id": 429, - "cpu_argmax": 5230, - "gpu_argmax": 5230, - "max_abs_diff": 0.2232283353805542, - "mean_abs_diff": 0.028549259528517723, - "cosine_similarity": 0.9999403953552246, - "kl_divergence": 0.001427050064290953, - "sigma_level": 109.33200148377001, - "cpk": 36.18388818749564, - "verdict": "Pass" - }, - { - "position": 68, - "token_id": 1030, - "cpu_argmax": 1012, - "gpu_argmax": 1012, - "max_abs_diff": 0.17638635635375977, - "mean_abs_diff": 0.02896297350525856, - "cosine_similarity": 0.9999390840530396, - "kl_divergence": 0.0001846553772036104, - "sigma_level": 108.80429517535535, - "cpk": 36.00549039858327, - "verdict": "Pass" - }, - { - "position": 69, - "token_id": 311, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.16861987113952637, - "mean_abs_diff": 0.024224400520324707, - "cosine_similarity": 0.9999402761459351, - "kl_divergence": 2.34391118189284e-6, - "sigma_level": 128.46318740442206, - "cpk": 42.561733826157166, - "verdict": "Pass" - }, - { - "position": 70, - "token_id": 1494, - "cpu_argmax": 1573, - "gpu_argmax": 1573, - "max_abs_diff": 0.20461726188659668, - "mean_abs_diff": 0.03377861529588699, - "cosine_similarity": 0.9999331831932068, - "kl_divergence": 0.00023387114648966236, - "sigma_level": 94.8402052044108, - "cpk": 31.346437500955034, - "verdict": "Pass" - }, - { - "position": 71, - "token_id": 1573, - "cpu_argmax": 279, - "gpu_argmax": 279, - "max_abs_diff": 0.1815171241760254, - "mean_abs_diff": 0.02938883751630783, - "cosine_similarity": 0.9999383687973022, - "kl_divergence": 0.00012852325512796255, - "sigma_level": 107.84635531283064, - "cpk": 35.68466185319231, - "verdict": "Pass" - }, - { - "position": 72, - "token_id": 279, - "cpu_argmax": 12801, - "gpu_argmax": 12801, - "max_abs_diff": 0.19804608821868896, - "mean_abs_diff": 0.039254080504179, - "cosine_similarity": 0.999911904335022, - "kl_divergence": 0.0006121267588050523, - "sigma_level": 83.23034839965909, - "cpk": 27.471188566847108, - "verdict": "Pass" - }, - { - "position": 73, - "token_id": 4879, - "cpu_argmax": 315, - "gpu_argmax": 315, - "max_abs_diff": 0.2028665542602539, - "mean_abs_diff": 0.03341694921255112, - "cosine_similarity": 0.9999181032180786, - "kl_divergence": 0.0005078500188547661, - "sigma_level": 96.64307108681912, - "cpk": 31.94523064575194, - "verdict": "Pass" - }, - { - "position": 74, - "token_id": 1410, - "cpu_argmax": 387, - "gpu_argmax": 387, - "max_abs_diff": 0.16966509819030762, - "mean_abs_diff": 0.024780509993433952, - "cosine_similarity": 0.999947726726532, - "kl_divergence": 0.0005655783300468226, - "sigma_level": 126.1360790464263, - "cpk": 41.78488331819689, - "verdict": "Pass" - }, - { - "position": 75, - "token_id": 387, - "cpu_argmax": 6509, - "gpu_argmax": 6509, - "max_abs_diff": 0.19145441055297852, - "mean_abs_diff": 0.02944508008658886, - "cosine_similarity": 0.9999592304229736, - "kl_divergence": 0.0004856583110845136, - "sigma_level": 107.8875111781858, - "cpk": 35.6977740254799, - "verdict": "Pass" - }, - { - "position": 76, - "token_id": 37113, - "cpu_argmax": 438, - "gpu_argmax": 438, - "max_abs_diff": 0.20439130067825317, - "mean_abs_diff": 0.028915509581565857, - "cosine_similarity": 0.9999307990074158, - "kl_divergence": 0.00023907793954586396, - "sigma_level": 110.44756254388363, - "cpk": 36.54971688521135, - "verdict": "Pass" - }, - { - "position": 77, - "token_id": 13, - "cpu_argmax": 576, - "gpu_argmax": 576, - "max_abs_diff": 0.24377059936523438, - "mean_abs_diff": 0.03568901866674423, - "cosine_similarity": 0.9998804926872253, - "kl_divergence": 0.0008035140327457511, - "sigma_level": 90.22054935175149, - "cpk": 29.80519287808895, - "verdict": "Pass" - } + { + "position": 0, + "token_id": 785, + "cpu_argmax": 914, + "gpu_argmax": 914, + "max_abs_diff": 0.22080087661743164, + "mean_abs_diff": 0.029157055541872978, + "cosine_similarity": 0.9999111890792847, + "kl_divergence": 0.00032579414447661754, + "sigma_level": 108.38003583659224, + "cpk": 35.863341718487604, + "verdict": "Pass" + }, + { + "position": 1, + "token_id": 3974, + "cpu_argmax": 13876, + "gpu_argmax": 13876, + "max_abs_diff": 0.1777787208557129, + "mean_abs_diff": 0.03252752497792244, + "cosine_similarity": 0.9999105930328369, + "kl_divergence": 3.7201543263529397e-06, + "sigma_level": 99.81459433877987, + "cpk": 33.000971303716966, + "verdict": "Pass" + }, + { + "position": 2, + "token_id": 13876, + "cpu_argmax": 38835, + "gpu_argmax": 38835, + "max_abs_diff": 0.17074143886566162, + "mean_abs_diff": 0.024813607335090637, + "cosine_similarity": 0.9999530911445618, + "kl_divergence": 6.065099658800252e-06, + "sigma_level": 126.29212369865193, + "cpk": 41.83622763563623, + "verdict": "Pass" + }, + { + "position": 3, + "token_id": 38835, + "cpu_argmax": 34208, + "gpu_argmax": 34208, + "max_abs_diff": 0.17793822288513184, + "mean_abs_diff": 0.029602359980344772, + "cosine_similarity": 0.9999439120292664, + "kl_divergence": 2.1107606354716864e-05, + "sigma_level": 106.68227318083109, + "cpk": 35.29758713909199, + "verdict": "Pass" + }, + { + "position": 4, + "token_id": 34208, + "cpu_argmax": 916, + "gpu_argmax": 916, + "max_abs_diff": 0.21715784072875977, + "mean_abs_diff": 0.033003006130456924, + "cosine_similarity": 0.9999298453330994, + "kl_divergence": 4.5147641864858906e-07, + "sigma_level": 97.43239515014821, + "cpk": 32.20950155551228, + "verdict": "Pass" + }, + { + "position": 5, + "token_id": 916, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.16768229007720947, + "mean_abs_diff": 0.026923540979623795, + "cosine_similarity": 0.9999296069145203, + "kl_divergence": 2.4167411428152043e-05, + "sigma_level": 115.93981600656502, + "cpk": 38.38647946990311, + "verdict": "Pass" + }, + { + "position": 6, + "token_id": 279, + "cpu_argmax": 15678, + "gpu_argmax": 15678, + "max_abs_diff": 0.1763458251953125, + "mean_abs_diff": 0.027678435668349266, + "cosine_similarity": 0.9999611973762512, + "kl_divergence": 1.2421132461965687e-06, + "sigma_level": 114.27627514935318, + "cpk": 37.8285093389394, + "verdict": "Pass" + }, + { + "position": 7, + "token_id": 15678, + "cpu_argmax": 5562, + "gpu_argmax": 5562, + "max_abs_diff": 0.2084965705871582, + "mean_abs_diff": 0.03252236545085907, + "cosine_similarity": 0.9999578595161438, + "kl_divergence": 1.444100421606142e-06, + "sigma_level": 98.27470173458114, + "cpk": 32.4918900978282, + "verdict": "Pass" + }, + { + "position": 8, + "token_id": 5562, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.2644195556640625, + "mean_abs_diff": 0.02975749969482422, + "cosine_similarity": 0.9999121427536011, + "kl_divergence": 0.0002420612186183089, + "sigma_level": 105.07405871764146, + "cpk": 34.764124466695144, + "verdict": "Pass" + }, + { + "position": 9, + "token_id": 1393, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.21603012084960938, + "mean_abs_diff": 0.03684907406568527, + "cosine_similarity": 0.9999308586120605, + "kl_divergence": 0.0006474370152407752, + "sigma_level": 86.82147331522204, + "cpk": 28.67388353001697, + "verdict": "Pass" + }, + { + "position": 10, + "token_id": 279, + "cpu_argmax": 8251, + "gpu_argmax": 8251, + "max_abs_diff": 0.2456355094909668, + "mean_abs_diff": 0.04065067693591118, + "cosine_similarity": 0.9999247193336487, + "kl_divergence": 0.001367731303832384, + "sigma_level": 78.56056947133877, + "cpk": 25.920728129656204, + "verdict": "Pass" + }, + { + "position": 11, + "token_id": 12801, + "cpu_argmax": 374, + "gpu_argmax": 374, + "max_abs_diff": 0.18509769439697266, + "mean_abs_diff": 0.03018302470445633, + "cosine_similarity": 0.9999426007270813, + "kl_divergence": 0.0005161167428984038, + "sigma_level": 104.15563778996108, + "cpk": 34.45656824761013, + "verdict": "Pass" + }, + { + "position": 12, + "token_id": 21926, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.17779541015625, + "mean_abs_diff": 0.031838010996580124, + "cosine_similarity": 0.999921977519989, + "kl_divergence": 0.00017258435469219227, + "sigma_level": 99.16622380548698, + "cpk": 32.792303324828296, + "verdict": "Pass" + }, + { + "position": 13, + "token_id": 35398, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.1694812774658203, + "mean_abs_diff": 0.026445960626006126, + "cosine_similarity": 0.9999368786811829, + "kl_divergence": 0.0005906903382915679, + "sigma_level": 118.52313191771943, + "cpk": 39.24650579924256, + "verdict": "Pass" + }, + { + "position": 14, + "token_id": 37402, + "cpu_argmax": 24258, + "gpu_argmax": 24258, + "max_abs_diff": 0.2208271026611328, + "mean_abs_diff": 0.0328725203871727, + "cosine_similarity": 0.9999487400054932, + "kl_divergence": 0.0007866923538006692, + "sigma_level": 96.84132305510252, + "cpk": 32.01515615449671, + "verdict": "Pass" + }, + { + "position": 15, + "token_id": 24258, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.18550491333007812, + "mean_abs_diff": 0.026621753349900246, + "cosine_similarity": 0.999907374382019, + "kl_divergence": 0.0002935938926612731, + "sigma_level": 117.17077206714823, + "cpk": 38.796983072900325, + "verdict": "Pass" + }, + { + "position": 16, + "token_id": 911, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.22534465789794922, + "mean_abs_diff": 0.035026710480451584, + "cosine_similarity": 0.9999264478683472, + "kl_divergence": 0.00047285014688066114, + "sigma_level": 90.85841261803856, + "cpk": 30.020931596555734, + "verdict": "Pass" + }, + { + "position": 17, + "token_id": 32168, + "cpu_argmax": 4802, + "gpu_argmax": 4802, + "max_abs_diff": 0.14307236671447754, + "mean_abs_diff": 0.02470676600933075, + "cosine_similarity": 0.999955952167511, + "kl_divergence": 3.484863188887505e-05, + "sigma_level": 128.68804300700253, + "cpk": 42.631058888436456, + "verdict": "Pass" + }, + { + "position": 18, + "token_id": 4802, + "cpu_argmax": 7079, + "gpu_argmax": 7079, + "max_abs_diff": 0.19958877563476562, + "mean_abs_diff": 0.028368674218654633, + "cosine_similarity": 0.9999251365661621, + "kl_divergence": 7.519966075083051e-05, + "sigma_level": 112.12267875571543, + "cpk": 37.1091619397265, + "verdict": "Pass" + }, + { + "position": 19, + "token_id": 5819, + "cpu_argmax": 5942, + "gpu_argmax": 5942, + "max_abs_diff": 0.1851489543914795, + "mean_abs_diff": 0.03547385707497597, + "cosine_similarity": 0.9999073147773743, + "kl_divergence": 0.0005868901702431452, + "sigma_level": 92.18467398700828, + "cpk": 30.455712499876228, + "verdict": "Pass" + }, + { + "position": 20, + "token_id": 11, + "cpu_argmax": 2670, + "gpu_argmax": 2670, + "max_abs_diff": 0.2207651138305664, + "mean_abs_diff": 0.03258340433239937, + "cosine_similarity": 0.9999098777770996, + "kl_divergence": 0.0009104736533284513, + "sigma_level": 96.75204615803823, + "cpk": 31.98797279934988, + "verdict": "Pass" + }, + { + "position": 21, + "token_id": 4237, + "cpu_argmax": 9471, + "gpu_argmax": 9471, + "max_abs_diff": 0.2389364242553711, + "mean_abs_diff": 0.038926344364881516, + "cosine_similarity": 0.9999305009841919, + "kl_divergence": 0.000779499423692475, + "sigma_level": 80.85147160964138, + "cpk": 26.688219517690097, + "verdict": "Pass" + }, + { + "position": 22, + "token_id": 23869, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.2731236219406128, + "mean_abs_diff": 0.04174664989113808, + "cosine_similarity": 0.9998652935028076, + "kl_divergence": 0.0016339679460959167, + "sigma_level": 74.70249389609015, + "cpk": 24.64094972714047, + "verdict": "Pass" + }, + { + "position": 23, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.24706506729125977, + "mean_abs_diff": 0.04125778004527092, + "cosine_similarity": 0.9998652935028076, + "kl_divergence": 0.00012628074726192903, + "sigma_level": 77.1027654306697, + "cpk": 25.43583106547151, + "verdict": "Pass" + }, + { + "position": 24, + "token_id": 15626, + "cpu_argmax": 14155, + "gpu_argmax": 14155, + "max_abs_diff": 0.23851919174194336, + "mean_abs_diff": 0.043194543570280075, + "cosine_similarity": 0.9997819066047668, + "kl_divergence": 0.0007928120095410873, + "sigma_level": 75.55819186163643, + "cpk": 24.914088819673893, + "verdict": "Pass" + }, + { + "position": 25, + "token_id": 49054, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1764129400253296, + "mean_abs_diff": 0.026423683390021324, + "cosine_similarity": 0.9998510479927063, + "kl_divergence": 3.542603986894582e-05, + "sigma_level": 118.8965940005014, + "cpk": 39.37039083716537, + "verdict": "Pass" + }, + { + "position": 26, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19931602478027344, + "mean_abs_diff": 0.03260093927383423, + "cosine_similarity": 0.9998522400856018, + "kl_divergence": 0.00016412521494609694, + "sigma_level": 98.43831627071472, + "cpk": 32.54534029265824, + "verdict": "Pass" + }, + { + "position": 27, + "token_id": 10272, + "cpu_argmax": 2022, + "gpu_argmax": 2022, + "max_abs_diff": 0.16838788986206055, + "mean_abs_diff": 0.025229420512914658, + "cosine_similarity": 0.9998432993888855, + "kl_divergence": 0.00022452636227061513, + "sigma_level": 124.04995771005096, + "cpk": 41.08917685771064, + "verdict": "Pass" + }, + { + "position": 28, + "token_id": 1506, + "cpu_argmax": 29728, + "gpu_argmax": 29728, + "max_abs_diff": 0.18775105476379395, + "mean_abs_diff": 0.028898026794195175, + "cosine_similarity": 0.9998990893363953, + "kl_divergence": 0.000461885527735682, + "sigma_level": 108.11494274168895, + "cpk": 35.77795520454613, + "verdict": "Pass" + }, + { + "position": 29, + "token_id": 6529, + "cpu_argmax": 23783, + "gpu_argmax": 23783, + "max_abs_diff": 0.1990138292312622, + "mean_abs_diff": 0.02971040830016136, + "cosine_similarity": 0.9999052882194519, + "kl_divergence": 0.00019171666737919304, + "sigma_level": 107.355287869335, + "cpk": 35.51929850346339, + "verdict": "Pass" + }, + { + "position": 30, + "token_id": 63515, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1763872504234314, + "mean_abs_diff": 0.02600252814590931, + "cosine_similarity": 0.9998703598976135, + "kl_divergence": 5.419791774745481e-06, + "sigma_level": 121.23067932375403, + "cpk": 40.14753442864604, + "verdict": "Pass" + }, + { + "position": 31, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.19364482164382935, + "mean_abs_diff": 0.027832137420773506, + "cosine_similarity": 0.9999036192893982, + "kl_divergence": 0.00015532038116852645, + "sigma_level": 112.43699543023251, + "cpk": 37.21821831774474, + "verdict": "Pass" + }, + { + "position": 32, + "token_id": 323, + "cpu_argmax": 1008, + "gpu_argmax": 1008, + "max_abs_diff": 0.1895672082901001, + "mean_abs_diff": 0.028663579374551773, + "cosine_similarity": 0.9999141097068787, + "kl_divergence": 0.00048805987877351045, + "sigma_level": 110.13832391530234, + "cpk": 36.44969475595692, + "verdict": "Pass" + }, + { + "position": 33, + "token_id": 279, + "cpu_argmax": 990, + "gpu_argmax": 990, + "max_abs_diff": 0.1705029010772705, + "mean_abs_diff": 0.02920815348625183, + "cosine_similarity": 0.9999076724052429, + "kl_divergence": 0.0006465286747198685, + "sigma_level": 107.00006736446687, + "cpk": 35.40624958893724, + "verdict": "Pass" + }, + { + "position": 34, + "token_id": 27889, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.17264866828918457, + "mean_abs_diff": 0.027726825326681137, + "cosine_similarity": 0.9998999238014221, + "kl_divergence": 9.778256838183266e-06, + "sigma_level": 113.50747960523304, + "cpk": 37.5735596967205, + "verdict": "Pass" + }, + { + "position": 35, + "token_id": 315, + "cpu_argmax": 32168, + "gpu_argmax": 32168, + "max_abs_diff": 0.20781350135803223, + "mean_abs_diff": 0.0317465104162693, + "cosine_similarity": 0.9999275803565979, + "kl_divergence": 0.0007944463203620559, + "sigma_level": 98.32937454807063, + "cpk": 32.516323639913914, + "verdict": "Pass" + }, + { + "position": 36, + "token_id": 656, + "cpu_argmax": 59711, + "gpu_argmax": 59711, + "max_abs_diff": 0.16794657707214355, + "mean_abs_diff": 0.025144487619400024, + "cosine_similarity": 0.9998830556869507, + "kl_divergence": 0.0014841781022014834, + "sigma_level": 124.04210451408106, + "cpk": 41.08745357459047, + "verdict": "Pass" + }, + { + "position": 37, + "token_id": 38589, + "cpu_argmax": 291, + "gpu_argmax": 291, + "max_abs_diff": 0.14394879341125488, + "mean_abs_diff": 0.024181175976991653, + "cosine_similarity": 0.99992835521698, + "kl_divergence": 4.590317076718935e-05, + "sigma_level": 131.6329585448339, + "cpk": 43.61239953703258, + "verdict": "Pass" + }, + { + "position": 38, + "token_id": 291, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.20485520362854004, + "mean_abs_diff": 0.032248660922050476, + "cosine_similarity": 0.9999155402183533, + "kl_divergence": 0.0010132197408340808, + "sigma_level": 97.76611926328172, + "cpk": 32.32597088527838, + "verdict": "Pass" + }, + { + "position": 39, + "token_id": 44378, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.17113685607910156, + "mean_abs_diff": 0.027429627254605293, + "cosine_similarity": 0.9999209046363831, + "kl_divergence": 0.00020635141816486075, + "sigma_level": 114.89910887906412, + "cpk": 38.03706631565145, + "verdict": "Pass" + }, + { + "position": 40, + "token_id": 3941, + "cpu_argmax": 5248, + "gpu_argmax": 5248, + "max_abs_diff": 0.20895957946777344, + "mean_abs_diff": 0.03152859956026077, + "cosine_similarity": 0.9999305605888367, + "kl_divergence": 0.0012345569716387572, + "sigma_level": 100.7526951257938, + "cpk": 33.31951576032809, + "verdict": "Pass" + }, + { + "position": 41, + "token_id": 3040, + "cpu_argmax": 2155, + "gpu_argmax": 2155, + "max_abs_diff": 0.2068023681640625, + "mean_abs_diff": 0.03304615244269371, + "cosine_similarity": 0.9999384880065918, + "kl_divergence": 0.0008537524363857488, + "sigma_level": 95.37599649244608, + "cpk": 31.52934802086009, + "verdict": "Pass" + }, + { + "position": 42, + "token_id": 97782, + "cpu_argmax": 821, + "gpu_argmax": 821, + "max_abs_diff": 0.19442176818847656, + "mean_abs_diff": 0.03274036571383476, + "cosine_similarity": 0.999926745891571, + "kl_divergence": 0.00022415574672985837, + "sigma_level": 96.70010563019095, + "cpk": 31.969535474822074, + "verdict": "Pass" + }, + { + "position": 43, + "token_id": 18432, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.1620619297027588, + "mean_abs_diff": 0.02630770020186901, + "cosine_similarity": 0.9999019503593445, + "kl_divergence": 0.00015874297246952666, + "sigma_level": 119.79814893017625, + "cpk": 39.670081827825925, + "verdict": "Pass" + }, + { + "position": 44, + "token_id": 26, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.2568702697753906, + "mean_abs_diff": 0.041561197489500046, + "cosine_similarity": 0.9999032020568848, + "kl_divergence": 0.0006645838048104513, + "sigma_level": 75.64519761529297, + "cpk": 24.953073788662536, + "verdict": "Pass" + }, + { + "position": 45, + "token_id": 1449, + "cpu_argmax": 13734, + "gpu_argmax": 13734, + "max_abs_diff": 0.23499298095703125, + "mean_abs_diff": 0.03136107325553894, + "cosine_similarity": 0.9999419450759888, + "kl_divergence": 0.0010564909150774027, + "sigma_level": 100.73015249089858, + "cpk": 33.31346702269047, + "verdict": "Pass" + }, + { + "position": 46, + "token_id": 14311, + "cpu_argmax": 304, + "gpu_argmax": 572, + "max_abs_diff": 0.19125032424926758, + "mean_abs_diff": 0.03027286008000374, + "cosine_similarity": 0.9999489188194275, + "kl_divergence": 0.0007324182535427381, + "sigma_level": 103.65502957340333, + "cpk": 34.290182007229085, + "verdict": "WarnArgmax" + }, + { + "position": 47, + "token_id": 572, + "cpu_argmax": 90326, + "gpu_argmax": 90326, + "max_abs_diff": 0.17596006393432617, + "mean_abs_diff": 0.028325680643320084, + "cosine_similarity": 0.9999551177024841, + "kl_divergence": 0.0004250593527600838, + "sigma_level": 111.41886548737655, + "cpk": 36.87662056233914, + "verdict": "Pass" + }, + { + "position": 48, + "token_id": 48826, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.1672215461730957, + "mean_abs_diff": 0.02670413628220558, + "cosine_similarity": 0.9999512434005737, + "kl_divergence": 0.0003977476078045197, + "sigma_level": 118.5229880052167, + "cpk": 39.24390816638344, + "verdict": "Pass" + }, + { + "position": 49, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.18988656997680664, + "mean_abs_diff": 0.029908394441008568, + "cosine_similarity": 0.999937117099762, + "kl_divergence": 0.00020613579089726636, + "sigma_level": 105.60082770042673, + "cpk": 34.9370799661288, + "verdict": "Pass" + }, + { + "position": 50, + "token_id": 1449, + "cpu_argmax": 11652, + "gpu_argmax": 11652, + "max_abs_diff": 0.18568384647369385, + "mean_abs_diff": 0.02834681235253811, + "cosine_similarity": 0.9999350309371948, + "kl_divergence": 0.0006665192974476394, + "sigma_level": 110.43670259979491, + "cpk": 36.55135682614585, + "verdict": "Pass" + }, + { + "position": 51, + "token_id": 1965, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.20452356338500977, + "mean_abs_diff": 0.0283601526170969, + "cosine_similarity": 0.9999455809593201, + "kl_divergence": 7.353835996759626e-05, + "sigma_level": 110.17008996001424, + "cpk": 36.46299327291264, + "verdict": "Pass" + }, + { + "position": 52, + "token_id": 572, + "cpu_argmax": 17256, + "gpu_argmax": 17256, + "max_abs_diff": 0.17694008350372314, + "mean_abs_diff": 0.029460342600941658, + "cosine_similarity": 0.999952495098114, + "kl_divergence": 0.000354042526327484, + "sigma_level": 108.24661766438967, + "cpk": 35.81645735131439, + "verdict": "Pass" + }, + { + "position": 53, + "token_id": 21870, + "cpu_argmax": 11, + "gpu_argmax": 11, + "max_abs_diff": 0.17319059371948242, + "mean_abs_diff": 0.025698617100715637, + "cosine_similarity": 0.9999451041221619, + "kl_divergence": 0.0012082822780696116, + "sigma_level": 121.84608568951509, + "cpk": 40.35442223805871, + "verdict": "Pass" + }, + { + "position": 54, + "token_id": 11, + "cpu_argmax": 323, + "gpu_argmax": 323, + "max_abs_diff": 0.18508267402648926, + "mean_abs_diff": 0.026767021045088768, + "cosine_similarity": 0.9999246597290039, + "kl_divergence": 8.510791162766133e-06, + "sigma_level": 117.67908059360465, + "cpk": 38.96386699563357, + "verdict": "Pass" + }, + { + "position": 55, + "token_id": 323, + "cpu_argmax": 1449, + "gpu_argmax": 1449, + "max_abs_diff": 0.23066043853759766, + "mean_abs_diff": 0.036110661923885345, + "cosine_similarity": 0.9999273419380188, + "kl_divergence": 0.0004994907989914782, + "sigma_level": 88.93142368056236, + "cpk": 29.3761935122759, + "verdict": "Pass" + }, + { + "position": 56, + "token_id": 279, + "cpu_argmax": 2197, + "gpu_argmax": 2197, + "max_abs_diff": 0.18856239318847656, + "mean_abs_diff": 0.030129026621580124, + "cosine_similarity": 0.9999247789382935, + "kl_divergence": 0.0008435059720457446, + "sigma_level": 104.45399344155254, + "cpk": 34.55573971808994, + "verdict": "Pass" + }, + { + "position": 57, + "token_id": 1895, + "cpu_argmax": 572, + "gpu_argmax": 572, + "max_abs_diff": 0.16795825958251953, + "mean_abs_diff": 0.0249911118298769, + "cosine_similarity": 0.9999393820762634, + "kl_divergence": 0.0001998253593928442, + "sigma_level": 124.82156616889559, + "cpk": 41.34723624638959, + "verdict": "Pass" + }, + { + "position": 58, + "token_id": 9482, + "cpu_argmax": 448, + "gpu_argmax": 448, + "max_abs_diff": 0.17892026901245117, + "mean_abs_diff": 0.028010983020067215, + "cosine_similarity": 0.9999496936798096, + "kl_divergence": 4.5661537655344266e-05, + "sigma_level": 112.21705530380771, + "cpk": 37.143742598712834, + "verdict": "Pass" + }, + { + "position": 59, + "token_id": 448, + "cpu_argmax": 264, + "gpu_argmax": 264, + "max_abs_diff": 0.23161697387695312, + "mean_abs_diff": 0.03567603975534439, + "cosine_similarity": 0.9999344348907471, + "kl_divergence": 0.0002818625074985845, + "sigma_level": 90.15571250675596, + "cpk": 29.78387093695512, + "verdict": "Pass" + }, + { + "position": 60, + "token_id": 264, + "cpu_argmax": 11682, + "gpu_argmax": 11682, + "max_abs_diff": 0.17027735710144043, + "mean_abs_diff": 0.026973217725753784, + "cosine_similarity": 0.9999549388885498, + "kl_divergence": 0.0005629856415008736, + "sigma_level": 116.05363658933409, + "cpk": 38.423683862478896, + "verdict": "Pass" + }, + { + "position": 61, + "token_id": 52573, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.1832141876220703, + "mean_abs_diff": 0.02868635207414627, + "cosine_similarity": 0.999939501285553, + "kl_divergence": 0.0006119540337820587, + "sigma_level": 109.6258088123588, + "cpk": 36.27987255845257, + "verdict": "Pass" + }, + { + "position": 62, + "token_id": 315, + "cpu_argmax": 1917, + "gpu_argmax": 1917, + "max_abs_diff": 0.2024369239807129, + "mean_abs_diff": 0.0438840389251709, + "cosine_similarity": 0.9999357461929321, + "kl_divergence": 0.00047113813301219134, + "sigma_level": 75.99682576801438, + "cpk": 25.054354617655378, + "verdict": "Pass" + }, + { + "position": 63, + "token_id": 52374, + "cpu_argmax": 69715, + "gpu_argmax": 69715, + "max_abs_diff": 0.18555545806884766, + "mean_abs_diff": 0.025885246694087982, + "cosine_similarity": 0.9999472498893738, + "kl_divergence": 0.000672849308967897, + "sigma_level": 121.69151806777218, + "cpk": 40.301338108777195, + "verdict": "Pass" + }, + { + "position": 64, + "token_id": 41017, + "cpu_argmax": 3589, + "gpu_argmax": 3589, + "max_abs_diff": 0.16499638557434082, + "mean_abs_diff": 0.02714230865240097, + "cosine_similarity": 0.9999595284461975, + "kl_divergence": 0.000296325939274267, + "sigma_level": 114.99287429648993, + "cpk": 38.070860424914805, + "verdict": "Pass" + }, + { + "position": 65, + "token_id": 22901, + "cpu_argmax": 9079, + "gpu_argmax": 3589, + "max_abs_diff": 0.17669677734375, + "mean_abs_diff": 0.02647567354142666, + "cosine_similarity": 0.9999456405639648, + "kl_divergence": 0.00043440203458147177, + "sigma_level": 119.37552964782644, + "cpk": 39.52846425329292, + "verdict": "WarnArgmax" + }, + { + "position": 66, + "token_id": 7354, + "cpu_argmax": 13, + "gpu_argmax": 13, + "max_abs_diff": 0.15739870071411133, + "mean_abs_diff": 0.0278735663741827, + "cosine_similarity": 0.999921977519989, + "kl_divergence": 0.0001710410126672348, + "sigma_level": 114.46856639915022, + "cpk": 37.89030153444305, + "verdict": "Pass" + }, + { + "position": 67, + "token_id": 429, + "cpu_argmax": 5230, + "gpu_argmax": 5230, + "max_abs_diff": 0.2232283353805542, + "mean_abs_diff": 0.028549259528517723, + "cosine_similarity": 0.9999403953552246, + "kl_divergence": 0.001427050064290953, + "sigma_level": 109.33200148377001, + "cpk": 36.18388818749564, + "verdict": "Pass" + }, + { + "position": 68, + "token_id": 1030, + "cpu_argmax": 1012, + "gpu_argmax": 1012, + "max_abs_diff": 0.17638635635375977, + "mean_abs_diff": 0.02896297350525856, + "cosine_similarity": 0.9999390840530396, + "kl_divergence": 0.0001846553772036104, + "sigma_level": 108.80429517535535, + "cpk": 36.00549039858327, + "verdict": "Pass" + }, + { + "position": 69, + "token_id": 311, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.16861987113952637, + "mean_abs_diff": 0.024224400520324707, + "cosine_similarity": 0.9999402761459351, + "kl_divergence": 2.34391118189284e-06, + "sigma_level": 128.46318740442206, + "cpk": 42.561733826157166, + "verdict": "Pass" + }, + { + "position": 70, + "token_id": 1494, + "cpu_argmax": 1573, + "gpu_argmax": 1573, + "max_abs_diff": 0.20461726188659668, + "mean_abs_diff": 0.03377861529588699, + "cosine_similarity": 0.9999331831932068, + "kl_divergence": 0.00023387114648966236, + "sigma_level": 94.8402052044108, + "cpk": 31.346437500955034, + "verdict": "Pass" + }, + { + "position": 71, + "token_id": 1573, + "cpu_argmax": 279, + "gpu_argmax": 279, + "max_abs_diff": 0.1815171241760254, + "mean_abs_diff": 0.02938883751630783, + "cosine_similarity": 0.9999383687973022, + "kl_divergence": 0.00012852325512796255, + "sigma_level": 107.84635531283064, + "cpk": 35.68466185319231, + "verdict": "Pass" + }, + { + "position": 72, + "token_id": 279, + "cpu_argmax": 12801, + "gpu_argmax": 12801, + "max_abs_diff": 0.19804608821868896, + "mean_abs_diff": 0.039254080504179, + "cosine_similarity": 0.999911904335022, + "kl_divergence": 0.0006121267588050523, + "sigma_level": 83.23034839965909, + "cpk": 27.471188566847108, + "verdict": "Pass" + }, + { + "position": 73, + "token_id": 4879, + "cpu_argmax": 315, + "gpu_argmax": 315, + "max_abs_diff": 0.2028665542602539, + "mean_abs_diff": 0.03341694921255112, + "cosine_similarity": 0.9999181032180786, + "kl_divergence": 0.0005078500188547661, + "sigma_level": 96.64307108681912, + "cpk": 31.94523064575194, + "verdict": "Pass" + }, + { + "position": 74, + "token_id": 1410, + "cpu_argmax": 387, + "gpu_argmax": 387, + "max_abs_diff": 0.16966509819030762, + "mean_abs_diff": 0.024780509993433952, + "cosine_similarity": 0.999947726726532, + "kl_divergence": 0.0005655783300468226, + "sigma_level": 126.1360790464263, + "cpk": 41.78488331819689, + "verdict": "Pass" + }, + { + "position": 75, + "token_id": 387, + "cpu_argmax": 6509, + "gpu_argmax": 6509, + "max_abs_diff": 0.19145441055297852, + "mean_abs_diff": 0.02944508008658886, + "cosine_similarity": 0.9999592304229736, + "kl_divergence": 0.0004856583110845136, + "sigma_level": 107.8875111781858, + "cpk": 35.6977740254799, + "verdict": "Pass" + }, + { + "position": 76, + "token_id": 37113, + "cpu_argmax": 438, + "gpu_argmax": 438, + "max_abs_diff": 0.20439130067825317, + "mean_abs_diff": 0.028915509581565857, + "cosine_similarity": 0.9999307990074158, + "kl_divergence": 0.00023907793954586396, + "sigma_level": 110.44756254388363, + "cpk": 36.54971688521135, + "verdict": "Pass" + }, + { + "position": 77, + "token_id": 13, + "cpu_argmax": 576, + "gpu_argmax": 576, + "max_abs_diff": 0.24377059936523438, + "mean_abs_diff": 0.03568901866674423, + "cosine_similarity": 0.9998804926872253, + "kl_divergence": 0.0008035140327457511, + "sigma_level": 90.22054935175149, + "cpk": 29.80519287808895, + "verdict": "Pass" + } ] + } } diff --git a/evidence/perf/3604/MEASUREMENT.md b/evidence/perf/3604/MEASUREMENT.md new file mode 100644 index 0000000000..5f21f7b6b6 --- /dev/null +++ b/evidence/perf/3604/MEASUREMENT.md @@ -0,0 +1,72 @@ +# #3604 — F2 guard receipt: before/after on the 144-word row + +**Box:** this dev box, `NVIDIA GeForce RTX 4090` (24564 MiB), `nvcc` at `/usr/bin/nvcc`. +**Binary:** `apr 0.68.2 (e6f77c98c)` built from this branch's working tree with +`CARGO_TARGET_DIR=/target-pinned` — pinned deliberately: the first build +went to the shared `/mnt/nvme-raid0/targets/aprender` and produced a binary from a +*different* worktree (`00e5d2dde`, no `--revalidate`), which is exactly the +stale-binary class the fleet was warned about the same evening. +**Model:** `/home/noah/models/Qwen3.5-4B-Q4_K_M.gguf`, 2,740,937,888 bytes, +sha256 `00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4`. +**Prompt:** 145 words (the "history of computing" paragraph), `--max-tokens 1`. +**Receipt dir:** `APR_F2_RECEIPT_DIR=/f2-receipts`, emptied before run A. +**GPU occupancy** (`nvidia-smi memory.used`) recorded immediately before every run. + +## The number (`done_when` 6) + +| run | page cache | GPU mem at start | wall | guard | receipt | +|---|---|---|---|---|---| +| A `--revalidate` | cold (first load) | 1639 MiB | 20.86 s | **9,752 ms**, 65 positions | written | +| B plain | warm | 1463 MiB | 7.35 s | **0** — skipped | read, sha256 1,173 ms | +| C plain | warm | 1467 MiB | 7.34 s | 0 | read, sha256 1,171 ms | +| **D `--revalidate`** | **warm** | 563 MiB | **17.18 s** | **9,747 ms**, 65 positions | rewritten | +| **E plain** | **warm** | 1639 MiB | **7.39 s** | **0** | read, sha256 1,173 ms | + +**D→E is the honest before/after** (same page-cache state): **17.18 s → 7.39 s, −9.8 s +wall**. The guard itself goes **9,747 ms → 0**. The key costs **1,173 ms** every run — +the whole-file sha256 at ~2.3 GB/s — so the *net* saving attributable to the guard is +**~8.6 s**, and the receipt line prints the hash cost separately so it cannot hide in +either number. A→B overstates the win (−13.5 s) because A also paid a cold page cache; +it is recorded, not claimed. + +The fresh-run guard cost (9.5–9.9 s across five fresh runs) matches the instrument +lane's 9,526–9,553 ms on lambda 4090 (#3598 row 1), so the "before" is the same +phenomenon, not a slower box. + +## The falsifiers (`done_when` 3, 4), end to end in the real binary + +Each planted receipt was perfect in two keys and wrong in the third; each run +**re-validated** (full 9.7 s guard, `[source=fresh]`) and **named the key**: + +| run | plant | guard line | +|---|---|---| +| F | `model_sha256` → `bbbb…` | `validating on this run (receipt is for model bbbbbbbbbbbb…, this file is 00fe7986ff5f…)` | +| G | `apr_version` → `0.61.0` | `validating on this run (receipt written by apr 0.61.0, this is 0.68.2)` | +| H | `device` → `NVIDIA GB10` | `validating on this run (receipt written for NVIDIA GB10, this device is NVIDIA GeForce RTX 4090)` | +| I | file overwritten with `{not json` | `validating on this run (receipt unreadable (…: not a receipt: …))` | +| J | file deleted | `validating on this run (no receipt for this model)` | + +Absence and unreadability both validate and are **distinguishable** from each other +(`done_when` 4). After J the receipt on disk reads `NVIDIA GeForce RTX 4090` again. + +`--revalidate` on a perfect receipt (runs A, D) validates and rewrites, reason +`--revalidate` (`done_when` 2). `done_when` 1 is B/C/E. + +## `done_when` 5 — partially, and stated + +`[source=receipt]` / `[source=fresh]` is on the guard's stderr line today, alongside +`sha256 ` and (fresh) `passed in `. The `apr run --json` field lands with +#3606's `StageTimings` (which is not on `main` yet): `F2Outcome { source, validate_ms, +sha256_ms, receipt_path }` is returned to the call site for exactly that purpose, so it +is one field on #3606's side, not a re-plumb. + +## What was NOT measured, and why + +- The CPU-reference vs GPU-probe split inside the 9.7 s — #3606 instruments it; out + of scope here by the ticket's own list. +- A cheaper identity than the whole-file sha256 (1.17 s/run). A candidate is a + follow-up with its own falsifier; the whole-file hash is the only identity under + which planted receipt F can be relied on to re-validate. + +Raw stderr for every run: `run_*.stderr.txt` in this directory; the receipt that run +D wrote: `receipt_example.json`. diff --git a/evidence/perf/3604/receipt_example.json b/evidence/perf/3604/receipt_example.json new file mode 100644 index 0000000000..f73a25614f --- /dev/null +++ b/evidence/perf/3604/receipt_example.json @@ -0,0 +1,8 @@ +{ + "schema": 1, + "model_sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4", + "apr_version": "0.68.2", + "device": "NVIDIA GeForce RTX 4090", + "validated_at": 1789941735, + "positions_judged": 65 +} \ No newline at end of file diff --git a/evidence/perf/3604/run_A_fresh_revalidate.stderr.txt b/evidence/perf/3604/run_A_fresh_revalidate.stderr.txt new file mode 100644 index 0000000000..8d129dcec4 --- /dev/null +++ b/evidence/perf/3604/run_A_fresh_revalidate.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (--revalidate) [source=fresh] +F2 guard: passed in 9752 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_B_receipt_hit.stderr.txt b/evidence/perf/3604/run_B_receipt_hit.stderr.txt new file mode 100644 index 0000000000..8f2eb7de5e --- /dev/null +++ b/evidence/perf/3604/run_B_receipt_hit.stderr.txt @@ -0,0 +1,2 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: receipt matches (model sha256 00fe7986ff5f…, apr 0.68.2, NVIDIA GeForce RTX 4090) — validated 8s ago on 65 positions; CPU reference forward skipped [source=receipt, sha256 1173 ms]. `apr run --revalidate` forces a fresh run. diff --git a/evidence/perf/3604/run_D_fresh_revalidate_WARM.stderr.txt b/evidence/perf/3604/run_D_fresh_revalidate_WARM.stderr.txt new file mode 100644 index 0000000000..5a1febd04a --- /dev/null +++ b/evidence/perf/3604/run_D_fresh_revalidate_WARM.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (--revalidate) [source=fresh] +F2 guard: passed in 9747 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_E_receipt_hit_WARM.stderr.txt b/evidence/perf/3604/run_E_receipt_hit_WARM.stderr.txt new file mode 100644 index 0000000000..da78ceafff --- /dev/null +++ b/evidence/perf/3604/run_E_receipt_hit_WARM.stderr.txt @@ -0,0 +1,2 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: receipt matches (model sha256 00fe7986ff5f…, apr 0.68.2, NVIDIA GeForce RTX 4090) — validated 7s ago on 65 positions; CPU reference forward skipped [source=receipt, sha256 1173 ms]. `apr run --revalidate` forces a fresh run. diff --git a/evidence/perf/3604/run_F_planted_wrong_sha256.stderr.txt b/evidence/perf/3604/run_F_planted_wrong_sha256.stderr.txt new file mode 100644 index 0000000000..c89f77700b --- /dev/null +++ b/evidence/perf/3604/run_F_planted_wrong_sha256.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (receipt is for model bbbbbbbbbbbb…, this file is 00fe7986ff5f…) [source=fresh] +F2 guard: passed in 9868 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_G_planted_wrong_apr_version.stderr.txt b/evidence/perf/3604/run_G_planted_wrong_apr_version.stderr.txt new file mode 100644 index 0000000000..fa22a02b30 --- /dev/null +++ b/evidence/perf/3604/run_G_planted_wrong_apr_version.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (receipt written by apr 0.61.0, this is 0.68.2) [source=fresh] +F2 guard: passed in 9780 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_H_planted_wrong_device.stderr.txt b/evidence/perf/3604/run_H_planted_wrong_device.stderr.txt new file mode 100644 index 0000000000..4223b1004d --- /dev/null +++ b/evidence/perf/3604/run_H_planted_wrong_device.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (receipt written for NVIDIA GB10, this device is NVIDIA GeForce RTX 4090) [source=fresh] +F2 guard: passed in 9947 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_I_corrupt_receipt.stderr.txt b/evidence/perf/3604/run_I_corrupt_receipt.stderr.txt new file mode 100644 index 0000000000..7d50e23fc7 --- /dev/null +++ b/evidence/perf/3604/run_I_corrupt_receipt.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (receipt unreadable (/f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json: not a receipt: key must be a string at line 1 column 2)) [source=fresh] +F2 guard: passed in 9724 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/perf/3604/run_J_missing_receipt.stderr.txt b/evidence/perf/3604/run_J_missing_receipt.stderr.txt new file mode 100644 index 0000000000..f09d34356d --- /dev/null +++ b/evidence/perf/3604/run_J_missing_receipt.stderr.txt @@ -0,0 +1,3 @@ +Backend: GPU (CUDA, NVIDIA GeForce RTX 4090, 24035 MB VRAM) [qwen35 hybrid forward, #3090] +F2 guard: validating on this run (no receipt for this model) [source=fresh] +F2 guard: passed in 9531 ms on 65 positions; receipt written to /f2-receipts/00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4.json — the next run of this (model, apr, device) skips it. diff --git a/evidence/verbs/refusals.json b/evidence/verbs/refusals.json new file mode 100644 index 0000000000..08fa293054 --- /dev/null +++ b/evidence/verbs/refusals.json @@ -0,0 +1,12 @@ +{ + "schema": "apr-refusal-ledger/v1", + "note": "The refusal bucket of the verb-set audit (#3597). Every entry is shaped by contracts/refusal-receipt-v1.yaml: a verb from the 111-verb registry, one reason line, the exit code a caller sees, and the release that removes the refusal or a declared sentinel saying none will. #3597 owns WHICH verbs land here; this file is seeded with the one refusal already measured so the shape has a real focus node rather than passing vacuously over an empty list.", + "refusals": [ + { + "verb": "bench", + "reason": "apr bench refuses qwen35 outright: the subcommand routes through the dense QuantizedGGUFTransformer loader, which does not carry the Gated-DeltaNet hybrid, so the benchmark verb cannot measure the architecture the last release shipped.", + "exit_code": 8, + "removed_by": "unscheduled" + } + ] +} diff --git a/scripts/check_baseline_ratchets.sh b/scripts/check_baseline_ratchets.sh index ddb9162967..40c27e21f2 100644 --- a/scripts/check_baseline_ratchets.sh +++ b/scripts/check_baseline_ratchets.sh @@ -93,7 +93,7 @@ classify() { # classify -> "[reason]", rc 1 if unclassifie cb200_baseline.txt) printf 'count\n' ;; # mirrors .pmat-gates.toml [tdg] baseline (PMAT-937) test_fixture_path_baseline.txt) printf 'count\n' ;; tracked_ignored_baseline.txt) printf 'count\n' ;; - unwired_guards_baseline.txt) printf 'set\n' ;; + unwired_guards_baseline.txt) printf 'set-aperture\tscripts/check_guards_are_wired.sh\n' ;; # NAME entries: a guard file that predates the comparand may be ledgered when the meta-guard itself widens (#3644) # NOT a ratchet either, and for the same reason one level along: this # registry is DERIVED from the test sources on every run # (scripts/check_tree_reader_tests.sh) and must equal that derivation @@ -323,6 +323,12 @@ if [ "${1:-}" = "--self-test" ] || [ "${1:-}" = "--selftest" ]; then # PREDATES; `fresh.md` is written by the working tree only. printf 'the old claim, 2.93 times faster\n' > "$SR/pre.md" printf 'GUARD v1\n' > "$SR/guard.sh" + # for the NAME-entry rows (#3644): an owner in a subdirectory with a + # sibling file, the shape of scripts/check_guards_are_wired.sh and + # its ledger of basenames + mkdir -p "$SR/sub" + printf 'OWNER v1\n' > "$SR/sub/owner.sh" + printf 'a sibling that predates\n' > "$SR/sub/sib.sh" git -C "$SR" add -A >/dev/null 2>&1 git -C "$SR" -c commit.gpgsign=false commit -qm 'aperture base' >/dev/null 2>&1 SR_APER=$(git -C "$SR" rev-parse HEAD) @@ -386,6 +392,28 @@ if [ "${1:-}" = "--self-test" ] || [ "${1:-}" = "--selftest" ]; then # guard edit at all -- otherwise the new kind would be strictly # WORSE than `set` on the ordinary path. ap_row 'aperture unchanged is green' 0 '# header\n' 'GUARD v1\n' 'the old claim, 2.93 times faster\n' + # -- NAME entries (#3644). unwired_guards_baseline.txt is a set of + # FILES, not of lines: the meta-guard's name:-line blindness hid + # four dark guards, and widening it reveals files already in the + # tree. (a) for a name is "the comparand carries the file" -- at the + # entry's path, or beside the owning guard, since the ledger names + # siblings by basename. Each admission has its refusal beside it. + ap_row 'aperture NAME entry that predates is admitted' 0 '# header\npre.md\n' 'GUARD v2\n' 'the old claim, 2.93 times faster\n' + ap_row 'aperture NAME entry this branch WROTE refuses' 1 '# header\nfresh.md\n' 'GUARD v2\n' 'the old claim, 2.93 times faster\n' + ap_row 'aperture NAME entry without a guard edit refuses' 1 '# header\npre.md\n' 'GUARD v1\n' 'the old claim, 2.93 times faster\n' + ap_row 'aperture NAME entry escaping the repo refuses' 1 '# header\n../pre.md\n' 'GUARD v2\n' 'the old claim, 2.93 times faster\n' + # beside the owner: `sib.sh` is not at the root, it is next to sub/owner.sh + printf '# header\nsib.sh\n' > "$SR/$P" + printf 'OWNER v2\n' > "$SR/sub/owner.sh" + ( BASELINE_RATCHET_BASE_REF="$SR_APER" \ + baseline_ratchet_check "$SR" "$P" set-aperture sub/owner.sh ) >/dev/null 2>&1 + say_row 'aperture NAME entry beside the OWNER is admitted' 0 $? + printf '# header\nsib.sh\n' > "$SR/$P" + printf 'GUARD v2\n' > "$SR/guard.sh" + ( BASELINE_RATCHET_BASE_REF="$SR_APER" \ + baseline_ratchet_check "$SR" "$P" set-aperture guard.sh ) >/dev/null 2>&1 + say_row 'aperture NAME entry beside a DIFFERENT owner refuses' 1 $? + printf 'OWNER v1\n' > "$SR/sub/owner.sh" # A missing owner argument must fail CLOSED. Called with no guard # path, "could not check" must never read as "no growth". printf '%b' "$AP_BASE" > "$SR/$P" diff --git a/scripts/check_fleet_pv_shapes_gate.sh b/scripts/check_fleet_pv_shapes_gate.sh index 788f95cad9..02b5540c0a 100755 --- a/scripts/check_fleet_pv_shapes_gate.sh +++ b/scripts/check_fleet_pv_shapes_gate.sh @@ -11,13 +11,26 @@ # runner and which binary produced the number. # # THREE OUTCOMES, EACH NAMED -- never a pass by silence: -# PASS pin declared, fleet pv present, capability probe passes, lint verdict Pass -# FAIL pin declared (this box says the tool should be here) but it is absent, cannot -# do `--gate`, or lint's verdict is not Pass -# UNMEASURED no pin: this runner was never converged (infra#708). Exit 0 with a row a -# consumer reads as not-measured. guard_tree.sh has no Unknown exit contract -# (measured: no guard emits one), so the row IS the verdict. First-green rule: -# it may not red a runner class that has never had the tool. +# PASS a verdict from the PINNED binary: lint's verdict is Pass (and pv's own +# planted-violation control fired, when pv reports it) +# FAIL a verdict from the PINNED binary: lint's verdict is not Pass, or pv's control +# did not fire. PASS and FAIL are reserved for a verdict from a binary that +# matches the pin, so a real shape violation still reds the gate. +# UNMEASURED FLEET STATE, not a verdict -- exit 0, with `reason=` naming which: +# no-pin this runner was never converged (infra#708) +# no-binary the pin declares pv and no fleet path has it (forjar drift) +# pin-mismatch the fleet binary is not the pinned version +# incapable the pinned binary cannot do `--gate` / `extract` +# pv-env the pinned binary returned no JSON verdict (e.g. it refused +# input records and exited 2) +# judge-env the verdict parser itself did not run (no python3, a crash) +# ANDON 2026-09-21 (main run 35563956223, intel-clean-room-2): intel's pin was +# rewritten to 0.65.2 by another repo's CI (paiml-implement#315) and every +# guard-tree on intel went RED with "the resolved binary is NOT the pinned one" +# -- fleet state a pull request cannot fix, reddening `gate` for every PR. A +# required gate may not red on fleet state (the first-green rule), so each of +# these is recorded, visibly: guard_tree.sh surfaces UNMEASURED lines under a +# passing guard (#3651), and the fleet fix belongs to the fleet. # # CAPABILITY, NOT VERSION. The probe asks `pv lint --help` for `--gate` and runs # `pv extract --help`; a version compare would have passed 0.65.2 on any `>= 0.6x` rule. @@ -28,7 +41,7 @@ # check_fleet_pv_shapes_gate.sh judge this runner # check_fleet_pv_shapes_gate.sh --self-test case table with stub pvs and a planted violation # -# Overrides, for the self-test only: FLEET_PV_BIN, FLEET_PV_PIN, FLEET_PV_CONTRACTS. +# Overrides, for the self-test only: FLEET_PV_BIN, FLEET_PV_PIN, FLEET_PV_CONTRACTS, FLEET_PV_PYTHON. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" || exit 2 # WHERE THE FLEET PUTS pv, from infra's forjar declarations (machines/*/forjar.yaml): @@ -41,6 +54,7 @@ FLEET_PV_CANDIDATES="${FLEET_PV_BIN:-/opt/fleet-bin/bin/pv:$HOME/.cargo/bin/pv}" PV_PIN="${FLEET_PV_PIN:-$HOME/.config/fleet/pv.pin}" CONTRACTS="${FLEET_PV_CONTRACTS:-$ROOT/contracts}" RUNNER="${RUNNER_NAME:-unknown}" # an Actions export; empty outside Actions, and empty is not absent +PV_PYTHON="${FLEET_PV_PYTHON:-python3}" # test seam: a dead interpreter reproduces judge-env rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } @@ -74,21 +88,21 @@ judge() { # GNU tr is merely lenient. No tr: bash strips the class itself, portably. pin=$(<"$PV_PIN"); pin=${pin//[[:space:]]/} if ! PV_BIN=$(resolve_fleet_pv); then - printf 'FAIL runner=%s pin=%s candidates=%s -- the pin declares the tool and none of the fleet paths has it; the box disagrees with its own declaration (forjar drift)\n' "$RUNNER" "$pin" "$FLEET_PV_CANDIDATES" >&2 - return 1 + printf 'UNMEASURED runner=%s reason=no-binary pin=%s candidates=%s -- the pin declares the tool and none of the fleet paths has it (forjar drift); fleet state, not a verdict\n' "$RUNNER" "$pin" "$FLEET_PV_CANDIDATES" + return 0 fi ver=$("$PV_BIN" --version 2>/dev/null | head -1 | awk '{print $2}') if [ "$ver" != "$pin" ]; then - printf 'FAIL runner=%s pin=%s pv=%s version=%s -- the resolved binary is NOT the pinned one; a number from it would be a number from the wrong tool (the 26-day-old apr class)\n' "$RUNNER" "$pin" "$PV_BIN" "${ver:-?}" >&2 - return 1 + printf 'UNMEASURED runner=%s reason=pin-mismatch pin=%s pv=%s version=%s -- the fleet binary is not the pinned one, so no number from it counts (the 26-day-old apr class); fleet state, not a verdict\n' "$RUNNER" "$pin" "$PV_BIN" "${ver:-?}" + return 0 fi if ! why=$(capable "$PV_BIN"); then - printf 'FAIL runner=%s pin=%s pv=%s version=%s -- %s; this is the 0.65.2 condition, the gate cannot run here\n' "$RUNNER" "$pin" "$PV_BIN" "${ver:-?}" "$why" >&2 - return 1 + printf 'UNMEASURED runner=%s reason=incapable pin=%s pv=%s version=%s -- %s (the 0.65.2 condition); fleet state, not a verdict\n' "$RUNNER" "$pin" "$PV_BIN" "${ver:-?}" "$why" + return 0 fi d=$(mktemp -d) || return 2 "$PV_BIN" lint "$CONTRACTS" --gate shapes --format json > "$d/lint.json" 2> "$d/lint.err"; rc=$? - out=$(python3 - "$d/lint.json" "$rc" <<'PY' + out=$("$PV_PYTHON" - "$d/lint.json" "$rc" <<'PY' import json, sys p, rc = sys.argv[1], int(sys.argv[2]) try: @@ -112,16 +126,21 @@ if v != "Pass": print("PASS " + row); sys.exit(0) PY ); rc=$? + local err1="" + [ -s "$d/lint.err" ] && err1=$(sed -n 1p "$d/lint.err" | cut -c1-200) rmtree "$d" case "$rc" in 0) printf '%s runner=%s pin=%s pv=%s version=%s\n' "$out" "$RUNNER" "$pin" "$PV_BIN" "$ver" ;; 1) printf '%s runner=%s pin=%s pv=%s version=%s\n' "$out" "$RUNNER" "$pin" "$PV_BIN" "$ver" >&2 ;; - *) printf '%s runner=%s pin=%s pv=%s version=%s\n' "$out" "$RUNNER" "$pin" "$PV_BIN" "$ver" >&2 ;; + 2) printf 'UNMEASURED runner=%s reason=pv-env pin=%s pv=%s version=%s -- %s; pv stderr: %s; fleet state, not a verdict\n' "$RUNNER" "$pin" "$PV_BIN" "$ver" "${out:-no output}" "${err1:-none}" + return 0 ;; + *) printf 'UNMEASURED runner=%s reason=judge-env pin=%s pv=%s version=%s -- the verdict parser (%s) did not run (rc=%s): %s; not a verdict\n' "$RUNNER" "$pin" "$PV_BIN" "$ver" "$PV_PYTHON" "$rc" "${out:-no output}" + return 0 ;; esac return "$rc" } -case "${1:-}" in -h|--help) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; esac +case "${1:-}" in -h|--help) sed -n '2,44p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; esac if [ "${1:-}" = "--self-test" ]; then echo "=== fleet pv shapes gate: case table (stub pvs, no network, no real fleet binary) ===" @@ -148,24 +167,26 @@ STUB mkpv "$d/pv_ok" capable Pass out=$(FLEET_PV_BIN="$d/pv_ok" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" RUNNER_NAME=probe-runner bash "$0" 2>&1); rc=$? [ "$rc" -eq 0 ] && grep -q '^PASS .*verdict=Pass.*runner=probe-runner' <<<"$out" && ok "capable pv + Pass verdict -> PASS, row names the runner" || nok "expected PASS row, got rc=$rc: $out" + # 1b. ...and re-prints it as SUMMARY, the line guard_tree.sh carries into the CI log (#3651) + [ "$rc" -eq 0 ] && grep -q '^SUMMARY PASS .*verdict=Pass.*runner=probe-runner.*version=' <<<"$out" && ok "a measured PASS is re-printed as SUMMARY for the guard-tree log" || nok "expected a SUMMARY PASS row, got rc=$rc: $out" # 2. PLANTED VIOLATION: lint says Fail -> the guard is RED (verdict parsed, not grepped) mkpv "$d/pv_fail" capable Fail out=$(FLEET_PV_BIN="$d/pv_fail" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$0" 2>&1); rc=$? [ "$rc" -eq 1 ] && grep -q '^FAIL .*verdict=Fail' <<<"$out" && ok "planted violation (verdict=Fail) -> RED" || nok "expected RED on verdict=Fail, got rc=$rc: $out" - # 3. the 0.65.2 condition: no --gate, no extract -> RED, named as a capability failure + # 3. the 0.65.2 condition: no --gate, no extract -> UNMEASURED reason=incapable (fleet state) mkpv "$d/pv_old" old Pass out=$(FLEET_PV_BIN="$d/pv_old" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$0" 2>&1); rc=$? - [ "$rc" -eq 1 ] && grep -q 'no --gate on lint' <<<"$out" && ok "0.65.2 condition (no --gate) -> RED by capability, not by version" || nok "expected capability RED, got rc=$rc: $out" + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED .*reason=incapable .*no --gate on lint' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "0.65.2 condition (no --gate) -> UNMEASURED reason=incapable, by capability not version, never PASS" || nok "expected UNMEASURED reason=incapable, got rc=$rc: $out" - # 4. pin declared, binary absent -> RED (the box disagrees with its own declaration) + # 4. pin declared, binary absent -> UNMEASURED reason=no-binary (forjar drift is fleet state) out=$(FLEET_PV_BIN="$d/does-not-exist" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$0" 2>&1); rc=$? - [ "$rc" -eq 1 ] && grep -q 'none of the fleet paths has it' <<<"$out" && ok "pin present, no fleet path has the binary -> RED" || nok "expected RED on absent binary, got rc=$rc: $out" + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED .*reason=no-binary .*none of the fleet paths has it' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "pin present, no fleet path has the binary -> UNMEASURED reason=no-binary, never PASS" || nok "expected UNMEASURED reason=no-binary, got rc=$rc: $out" # 5. no pin -> UNMEASURED, exit 0, the row says so and names the runner and infra#708 out=$(FLEET_PV_BIN="$d/pv_ok" FLEET_PV_PIN="$d/no-such-pin" FLEET_PV_CONTRACTS="$d/contracts" RUNNER_NAME=never-converged bash "$0" 2>&1); rc=$? - [ "$rc" -eq 0 ] && grep -q '^UNMEASURED runner=never-converged.*infra#708' <<<"$out" && ! grep -q '^PASS' <<<"$out" && ok "no pin -> UNMEASURED row (exit 0, never a PASS line)" || nok "expected UNMEASURED row, got rc=$rc: $out" + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED runner=never-converged.*infra#708' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "no pin -> UNMEASURED row (exit 0, never a PASS line)" || nok "expected UNMEASURED row, got rc=$rc: $out" # 6. pv's own control silent (pc_shape != fired) -> RED even with verdict=Pass mkpv "$d/pv_silent" capable Pass silent @@ -177,10 +198,41 @@ STUB out=$(FLEET_PV_BIN="$d/does-not-exist:$d/pv_second" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$0" 2>&1); rc=$? [ "$rc" -eq 0 ] && grep -q "^PASS .*pv=$d/pv_second" <<<"$out" && ok "first fleet path absent, second present -> resolved to the second, row names it" || nok "expected PASS via second candidate, got rc=$rc: $out" - # 8. version != pin -> RED: the resolved binary is not the pinned one (the 26-day-old apr class) + # 8. version != pin -> UNMEASURED reason=pin-mismatch: no number from the wrong tool counts, + # and the mismatch is fleet state (ANDON 2026-09-21: intel's pin rewritten to 0.65.2 by + # paiml-implement#315 redded every intel guard-tree on main, run 35563956223) mkpv "$d/pv_stale" capable Pass fired 0.65.2 - out=$(FLEET_PV_BIN="$d/pv_stale" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$0" 2>&1); rc=$? - [ "$rc" -eq 1 ] && grep -q 'NOT the pinned one' <<<"$out" && ok "binary 0.65.2 under pin 0.68.2 -> RED, named as a pin mismatch (before any lint)" || nok "expected pin-mismatch RED, got rc=$rc: $out" + out=$(FLEET_PV_BIN="$d/pv_stale" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" RUNNER_NAME=intel-clean-room-2 bash "$0" 2>&1); rc=$? + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED runner=intel-clean-room-2 reason=pin-mismatch pin=0.68.2 .*version=0.65.2' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "binary 0.65.2 under pin 0.68.2 -> UNMEASURED reason=pin-mismatch, names both versions and the runner, never PASS (before any lint)" || nok "expected UNMEASURED reason=pin-mismatch, got rc=$rc: $out" + # ...the mutant that restores FAIL must turn that row RED: the row discriminates + sed '/printf .UNMEASURED runner=%s reason=pin-mismatch pin=%s/{n;s/return 0/return 1/;}' "$0" > "$d/mut_pin.sh" + if cmp -s "$0" "$d/mut_pin.sh"; then nok "mutant (pin-mismatch -> FAIL) did not apply; the discrimination row proves nothing" + else out=$(FLEET_PV_BIN="$d/pv_stale" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$d/mut_pin.sh" 2>&1); rc=$? + [ "$rc" -ne 0 ] && ok "mutant restoring FAIL on pin-mismatch exits $rc -- the pin-mismatch row would go RED under it" || nok "mutant restoring FAIL still exits 0: $out"; fi + + # 8b. the fleet binary matches the pin but returns NO JSON (it refused input records and + # exited 2 -- measured on lambda with pv 0.68.2 against main's un-migrated parity + # records) -> UNMEASURED reason=pv-env, carrying pv's own stderr line + cat > "$d/pv_env" <<'STUB' +#!/usr/bin/env bash +case "$1" in + --version) echo "pv 0.68.2 (stub)";; + lint) case "$2" in --help) echo "--gate gate to run";; *) echo "shapes: refused evidence/parity/x.json: an UNMIGRATED logit-parity record" >&2; exit 2;; esac;; + extract) exit 0;; +esac +STUB + chmod 755 "$d/pv_env" + out=$(FLEET_PV_BIN="$d/pv_env" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" RUNNER_NAME=probe-runner bash "$0" 2>&1); rc=$? + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED runner=probe-runner reason=pv-env .*no JSON verdict .*UNMIGRATED logit-parity record' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "pinned pv exits 2 with no JSON -> UNMEASURED reason=pv-env with pv's stderr, never PASS" || nok "expected UNMEASURED reason=pv-env, got rc=$rc: $out" + # ...and its mutant: restore the ENV exit and the row goes RED + sed 's/^\( *\)2) printf .UNMEASURED runner=%s reason=pv-env/\1 2) return 2; printf "unreached/' "$0" > "$d/mut_env.sh" + if cmp -s "$0" "$d/mut_env.sh"; then nok "mutant (pv-env -> ENV exit) did not apply; the discrimination row proves nothing" + else out=$(FLEET_PV_BIN="$d/pv_env" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" bash "$d/mut_env.sh" 2>&1); rc=$? + [ "$rc" -ne 0 ] && ok "mutant restoring the ENV exit on pv-env exits $rc -- the pv-env row would go RED under it" || nok "mutant restoring the ENV exit still exits 0: $out"; fi + + # 8c. the verdict parser cannot run (no python3 on the runner) -> UNMEASURED reason=judge-env + out=$(FLEET_PV_BIN="$d/pv_ok" FLEET_PV_PIN="$d/pin" FLEET_PV_CONTRACTS="$d/contracts" FLEET_PV_PYTHON="$d/no-such-python" bash "$0" 2>&1); rc=$? + [ "$rc" -eq 0 ] && grep -q '^UNMEASURED .*reason=judge-env .*rc=127' <<<"$out" && ! grep -qE '^(SUMMARY )?PASS' <<<"$out" && ok "no python3 for the verdict parser -> UNMEASURED reason=judge-env (rc=127), never PASS" || nok "expected UNMEASURED reason=judge-env, got rc=$rc: $out" # 9. pv omits pc_shape entirely -> PASS with control=unreported in the row (not a false negative) mkpv "$d/pv_nopc" capable Pass omit @@ -200,7 +252,15 @@ echo "=== shapes gate on the FLEET-PINNED pv, on this runner (check_fleet_pv_sha # The row is the verdict. No trailer may begin with PASS unless the row did: a trailer # reading "PASS-OR-UNMEASURED" is exactly what a consumer's grep would misread (the # self-test's row 5 refused it). +# A measured PASS is re-printed as `SUMMARY `: guard_tree.sh surfaces SUMMARY +# and UNMEASURED lines under a passing guard (#3651), so the runner, pv, version, +# verdict and corpus reach the CI log instead of a bare "PASS" (#3567). UNMEASURED +# needs no SUMMARY -- guard_tree surfaces it unasked. row=$(judge 2>&1); rc=$? printf '%s\n' "$row" -case "$rc" in 0) printf '%s\n' "${row%% *}";; 1) echo "FAIL" >&2;; *) echo "ENV (rc=$rc)" >&2;; esac +case "$rc" in + 0) case "$row" in "PASS "*) printf 'SUMMARY %s\n' "$row" ;; *) printf '%s\n' "${row%% *}" ;; esac ;; + 1) echo "FAIL" >&2 ;; + *) echo "ENV (rc=$rc)" >&2 ;; +esac exit "$rc" diff --git a/scripts/check_guards_are_wired.sh b/scripts/check_guards_are_wired.sh index 2d581ed8f1..309dddedc0 100755 --- a/scripts/check_guards_are_wired.sh +++ b/scripts/check_guards_are_wired.sh @@ -66,11 +66,25 @@ BASELINE="${REPO_ROOT}/scripts/unwired_guards_baseline.txt" # committed by the very guard that exists to catch it. An argument-wired guard # is still named by its own workflow and is found by the scan below on that # account; a release-time guard is deliberately unwired and stays reported. +# A STEP NAME IS NOT AN INVOCATION (#3644). ci.yml carries +# - name: "guard_tree.sh's own case table (BSE-01, wired BSE-02)" +# - name: "guard_tree.sh's parallel dispatcher case table (PMAT-1098)" +# The invocation regex below accepts `guard_tree.sh` followed by `'` -- meant +# for a quoted `run: "..."` -- so the apostrophe in `guard_tree.sh's` matched, +# neither line says --no-cargo, and each was read as a BARE dispatch in mode +# "all": the full --dry-run RUN set, 55 cargo-classified guards that +# `--no-cargo` never runs, all counted wired. Four were dark behind it, +# including the #3305 publish-strip guard, and this meta-guard said PASS. +# Proven by rewriting those two names (nothing else): 3 -> 7 unwired. +# Mention-vs-execution, one level below the trailing-comment case above. +# A `name:` line is documentation whatever it contains; drop it before matching. +_not_a_name_line() { grep -vE '^[[:space:]]*-?[[:space:]]*name:' || true; } + dispatcher_wired() { local root="$1" lines modes mode flag out [ -f "$root/scripts/guard_tree.sh" ] || return 0 lines=$(grep -rh --include='*.yml' --include='*.yaml' -- 'guard_tree.sh' \ - "$root"/.github/workflows/ 2>/dev/null | sed 's/#.*$//') || lines='' + "$root"/.github/workflows/ 2>/dev/null | sed 's/#.*$//' | _not_a_name_line) || lines='' # Invocation, not mention -- same test as the scan below. lines=$(grep -E "(^|[[:space:];&|(])((ba)?sh[[:space:]]+|\\./)?[^[:space:]]*guard_tree\\.sh([[:space:]]|$|['\"])" \ <<< "$lines") || lines='' @@ -154,7 +168,7 @@ unwired_in() { local mentions mentions=$(grep -rh --include='*.yml' --include='*.yaml' -- "$base" \ "$root"/.github/workflows/ 2>/dev/null \ - | sed 's/#.*$//') || mentions='' + | sed 's/#.*$//' | _not_a_name_line) || mentions='' if ! grep -qE "(^|[[:space:];&|(])((ba)?sh[[:space:]]+|\\./)?[^[:space:]]*${base}([[:space:]]|$|['\"])" <<< "$mentions" ; then printf '%s\n' "$base" fi @@ -272,8 +286,39 @@ if [ "${1:-}" = "--self-test" ]; then printf 'FAIL row 7 got [%s], expected [check_nightly_only.sh check_nowhere.sh ]\n' "$got7"; fails=1 fi + # ── Rows 8-10: A STEP NAME IS NOT AN INVOCATION (#3644) ───────────────── + # + # Row 8 is the exact ci.yml shape that hid four guards: a step NAMED + # `guard_tree.sh's ...`, no dispatcher run: line. The dispatched fixture + # guard must come back as unwired. Row 9 is the control the `'` in the + # regex exists for: a QUOTED run: line still dispatches. Row 10 is the same + # blindness in the per-guard scan: a step named after a guard wires nothing. + printf 'jobs:\n gate:\n steps:\n - name: "guard_tree.sh'"'"'s own case table (BSE-01, wired BSE-02)"\n run: echo a name is not a dispatch\n' \ + > "$TD2/.github/workflows/ci.yml" + got8=$(unwired_in "$TD2" | tr '\n' ' ') + if [ "$got8" = "check_dispatched.sh " ]; then + printf 'ok row 8 a step NAMED guard_tree.sh'"'"'s ... is not a dispatch; the guard stays unwired\n' + else + printf 'FAIL row 8 got [%s], expected [check_dispatched.sh ] -- a step name read as a bare dispatch\n' "$got8"; fails=1 + fi + printf 'jobs:\n gate:\n steps:\n - run: "bash scripts/guard_tree.sh --no-cargo"\n' \ + > "$TD2/.github/workflows/ci.yml" + if [ -z "$(unwired_in "$TD2")" ]; then + printf 'ok row 9 a QUOTED run: line still dispatches (the control for row 8)\n' + else + printf 'FAIL row 9 quoted dispatch not honoured: [%s]\n' "$(unwired_in "$TD2" | tr '\n' ' ')"; fails=1 + fi + printf 'jobs:\n x:\n steps:\n - name: "check_dark.sh'"'"'s job"\n run: bash scripts/check_wired.sh\n' \ + > "$TD/.github/workflows/ci.yml" + got10=$(unwired_in "$TD" | tr '\n' ' ') + if [ "$got10" = "check_dark.sh " ]; then + printf 'ok row 10 a step named after a guard wires nothing\n' + else + printf 'FAIL row 10 got [%s], expected [check_dark.sh ]\n' "$got10"; fails=1 + fi + [ "$fails" -eq 0 ] || { printf '\nSELF-TEST FAILED\n'; exit 1; } - printf '\nSELF-TEST PASSED (7/7)\n' + printf '\nSELF-TEST PASSED (10/10)\n' exit 0 fi @@ -332,7 +377,11 @@ fi # branch cannot rewrite, and never the branch against itself. # shellcheck source=scripts/lib_baseline_ratchet.sh . "${REPO_ROOT}/scripts/lib_baseline_ratchet.sh" || exit 1 -baseline_ratchet_check "${REPO_ROOT}" scripts/unwired_guards_baseline.txt set || exit 1 +# set-aperture, owned by this file (#3644): when THIS guard widens and reveals +# guards that were already dark, the ledger may record them -- only guards the +# comparand already carries, only in a diff that changes this file, every +# admission printed. On every other PR it is `set`: shrink-only. +baseline_ratchet_check "${REPO_ROOT}" scripts/unwired_guards_baseline.txt set-aperture scripts/check_guards_are_wired.sh || exit 1 if [ ! -f "$BASELINE" ]; then printf 'FAIL: %s missing. Run --update once to establish it.\n' "$BASELINE" diff --git a/scripts/check_model_parity.sh b/scripts/check_model_parity.sh index 4a75d10585..1e3927f33c 100755 --- a/scripts/check_model_parity.sh +++ b/scripts/check_model_parity.sh @@ -39,7 +39,13 @@ if "min_cosine" not in rule or not str(rule.get("basis") or "").strip(): mc = float(rule["min_cosine"]); basis = str(rule["basis"]) try: d = json.load(open(f, encoding="utf-8")) except Exception as e: print(f"FAIL {model}: unreadable apr parity output ({e})"); sys.exit(1) -rows = d.get("metrics") if isinstance(d, dict) else None +# A v2 receipt (apr-parity-receipt/v2, #3577) EMBEDS the raw `apr parity --json` document under +# `raw`, so the readings live at `raw.metrics`. A fresh `apr parity` run is the raw document itself +# and carries them at the top level. Those are two different INPUTS — a tool's output and an archived +# receipt that quotes it — not two spellings of one, so the rule is stated once: look inside the +# envelope when there is one. Anything else is still refused by the line below. +raw = d.get("raw") if isinstance(d, dict) and isinstance(d.get("raw"), dict) else d +rows = raw.get("metrics") if isinstance(raw, dict) else None if not isinstance(rows, list) or not rows: print(f"FAIL {model}: no per-position metrics in the output"); sys.exit(1) cos = [(r.get("position"), float(r.get("cosine_similarity"))) for r in rows if r.get("cosine_similarity") is not None] if len(cos) < minpos: print(f"FAIL {model}: {len(cos)} positions < min_positions {minpos} (I8: an autoregressive gate validates over >= 64 positions)"); sys.exit(1) @@ -172,9 +178,16 @@ if [ "${1:-}" = "--self-test" ]; then row 0 "the gx10 7B record is GREEN (min 0.9985)" judge "$G/qwen2.5-coder-7b-instruct-q4_k_m.json" qwen2.5-coder-7b-instruct python3 - "$L/qwen2.5-coder-7b-instruct-q4_k_m.json" "$TD/twin.json" "$TD/short.json" <<'PY' import json, sys -d = json.load(open(sys.argv[1])); m = d["metrics"] -m[40]["cosine_similarity"] = 0.5; json.dump(d, open(sys.argv[2], "w")) # must-RED twin: one position at 0.5 -d2 = json.load(open(sys.argv[1])); d2["metrics"] = d2["metrics"][:20]; json.dump(d2, open(sys.argv[3], "w")) # fewer than 64 positions +# The committed records are apr-parity-receipt/v2 (#3577): the raw `apr parity --json` document is +# embedded under `raw`. Mutate the readings where they actually live, or the twin is built from a +# KeyError and the must-RED control never fires. +def rows(doc): return doc["raw"]["metrics"] if isinstance(doc.get("raw"), dict) else doc["metrics"] +def put(doc, v): + (doc["raw"] if isinstance(doc.get("raw"), dict) else doc)["metrics"] = v + return doc +d = json.load(open(sys.argv[1])); rows(d)[40]["cosine_similarity"] = 0.5 +json.dump(d, open(sys.argv[2], "w")) # must-RED twin: one position at 0.5 +d2 = json.load(open(sys.argv[1])); json.dump(put(d2, rows(d2)[:20]), open(sys.argv[3], "w")) # fewer than 64 positions PY [ -f "$ROOT/tests/fixtures/parity/defective/one-position-at-0.5.json" ] || cp "$TD/twin.json" "$ROOT/tests/fixtures/parity/defective/one-position-at-0.5.json" row 1 "the must-RED twin (7B with position 40 forced to 0.5) is RED naming the position" judge "$TD/twin.json" qwen2.5-coder-7b-instruct diff --git a/scripts/check_pathonly_devdeps_unused_in_src.sh b/scripts/check_pathonly_devdeps_unused_in_src.sh index c6ae4812a8..092016d44f 100755 --- a/scripts/check_pathonly_devdeps_unused_in_src.sh +++ b/scripts/check_pathonly_devdeps_unused_in_src.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # check_pathonly_devdeps_unused_in_src.sh - src/ may not reference a dev-dep -# that `cargo publish` deletes. +# that publishing to crates.io deletes. # # THE CLASS. A dev-dependency written as `{ path = "..." }` with no version, -# no git and no workspace inheritance carries NO SOURCE. `cargo publish` omits +# no git and no workspace inheritance carries NO SOURCE. The publish step omits # it from the published manifest entirely -- that omission is deliberate and is # what lets a dev-dep cycle (aprender-compute -> aprender-core) be legal. But # the crate's own `#[cfg(test)]` code is published with it. If that code names @@ -29,8 +29,30 @@ # baseline fails, and a baseline pair with no remaining violation also fails, so # a fix must delete its row in the same commit. Draining it to empty is #3306. # +# WHO RUNS IT, AND WHY THIS FILE NEVER WRITES THE BUILD TOOL'S NAME WITH A SPACE +# AFTER IT (#3644). guard_tree.sh classifies a guard as cargo-using by the +# substring CARGO_RE, comments included; `guard-tree` runs only the cargo-free +# population (--no-cargo) and `guard-cargo` names its guards by hand. This file +# said the tool's name in three comments and one FAIL string, was classified +# cargo-using, was named by nothing, and RAN NOWHERE from the day it was +# written (2026-09-15) -- while check_guards_are_wired.sh reported PASS, blinded +# by a step name. It invokes no build tool: python over the manifests, then +# grep. Now it is in the --no-cargo population and guard-tree runs it. +# +# ENV IS NEVER A CODE VERDICT (#3644, the same shape as #3626's guard). The +# scanner needs a TOML reader: tomllib (python 3.11+) or tomli. The intel +# runner host has python 3.10.12 and neither; the old module-level import died +# with a traceback, `out=$(scan ... || true)` swallowed it, and the case table +# said "FAIL: missed hit_use" -- the regex reported broken by an interpreter +# that never ran it. Now the scanner ends with a `SCAN-DONE manifests=N` +# trailer that the shell REQUIRES; anything without it (no reader, a dead +# interpreter, no python3) is ENV rc=2 naming the interpreter and the runner -- +# never rc=1, never "no violations". +# # Runs bare, no arguments, from anywhere in the repo. `--selftest` runs the case # table only. A bare run does BOTH: the case table first, then the tree. +# PATHONLY_GUARD_PYTHON= test seam: a dead one reproduces the intel shape +# PATHONLY_GUARD_FORCE_NO_TOML=1 test seam: both readers fail to import, for real set -euo pipefail REPO="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" @@ -47,13 +69,33 @@ cleanup() { } trap cleanup EXIT -# scan -> prints "manifest|alias|src-file:line:text" per violation +# scan -> prints "manifest|alias|src-file:line:text" per violation. +# rc 0 with the trailer consumed; rc 2 (ENV) when the scanner did not finish. scan() { - python3 - "$1" <<'PY' -import os, re, sys, tomllib + local out rc=0 interp="${PATHONLY_GUARD_PYTHON:-python3}" + out=$(PATHONLY_GUARD_RUNNER="${RUNNER_NAME:-unknown}" "$interp" - "$1" 2>&1 <<'PY' +import os, re, sys + +runner = os.environ.get("PATHONLY_GUARD_RUNNER", "unknown") +if os.environ.get("PATHONLY_GUARD_FORCE_NO_TOML") == "1": + sys.modules["tomllib"] = None # the imports below then raise for real + sys.modules["tomli"] = None + +toml = None +for name in ("tomllib", "tomli"): + try: + toml = __import__(name) + break + except ImportError: + continue +if toml is None: + print("ENV: no TOML reader on this interpreter (python %d.%d, need tomllib >= 3.11 or tomli) runner=%s -- " + "the manifests were not read; this is not 'no violations'" % (sys.version_info[0], sys.version_info[1], runner)) + sys.exit(2) root = sys.argv[1] KINDS = ("dev-dependencies",) +manifests = 0 def sourceless(spec): """True when the dep carries a path and nothing that survives publish.""" @@ -73,9 +115,10 @@ for dirpath, dirnames, filenames in os.walk(root): continue try: with open(manifest, "rb") as fh: - data = tomllib.load(fh) - except (tomllib.TOMLDecodeError, OSError): + data = toml.load(fh) + except (toml.TOMLDecodeError, OSError): continue + manifests += 1 aliases = [a for k in KINDS for a, s in (data.get(k) or {}).items() if sourceless(s)] if not aliases: continue @@ -100,7 +143,21 @@ for dirpath, dirnames, filenames in os.walk(root): rel_m = os.path.relpath(manifest, root) rel_s = os.path.relpath(path, root) print(f"{rel_m}|{alias}|{rel_s}:{n}:{stripped[:100]}") +# positive evidence that the scan RAN TO THE END; the shell refuses any output without it +print("SCAN-DONE manifests=%d reader=%s" % (manifests, toml.__name__)) PY + ) || rc=$? + case "$out" in + *"SCAN-DONE manifests="*) ;; + *) + printf 'ENV: the scanner did not finish (interpreter=%s rc=%s runner=%s) -- not a pass, not a code verdict\n' \ + "$interp" "$rc" "${RUNNER_NAME:-unknown}" >&2 + [ -z "$out" ] || printf '%s\n' "$out" | sed 's/^/ /' >&2 + return 2 ;; + esac + # the trailer is evidence, not a row + printf '%s\n' "$out" | grep -v '^SCAN-DONE ' || true + return 0 } # ── case table: every row is a fixture the regex/selector must classify ── @@ -125,7 +182,7 @@ selftest() { mk hit_underscore 'two-words = { path = "../tw" }' 'use two_words::x;' mk hit_extern 'sib = { path = "../sib" }' 'extern crate sib;' - # MUST NOT MATCH -- these all survive `cargo publish`, or are not a reference + # MUST NOT MATCH -- these all survive publishing, or are not a reference mk ok_versioned 'sib = { path = "../sib", version = "0.1.0" }' 'use sib::thing;' mk ok_workspace 'sib = { workspace = true }' 'use sib::thing;' mk ok_git 'sib = { git = "https://x/y", path = "../sib" }' 'use sib::thing;' @@ -134,7 +191,14 @@ selftest() { mk ok_comment 'sib = { path = "../sib" }' '// use sib::thing;' mk ok_substring 'sib = { path = "../sib" }' 'use sibling::thing;' - out="$(scan "$tmp" || true)" + # ENV from the scanner is ENV here: rc 2, never "missed hit_use" (#3644) + local env_rc=0 + out="$(scan "$tmp")" || env_rc=$? + if [ "$env_rc" -ne 0 ]; then + printf 'ENV: the case table could not be measured (scan rc=%s)\n' "$env_rc" + cleanup; FIXTURES="" + return 2 + fi # Piping a producer into a quiet grep is banned here # (check_no_pipe_into_grep_q.sh): grep exits on its first match and the @@ -156,18 +220,54 @@ selftest() { for row in ok_versioned ok_workspace ok_git ok_registry ok_unused ok_comment ok_substring; do if row_present "$row"; then printf 'FAIL: false positive on %s\n' "$row"; rc=1; fi done + + # THE DEATH ROWS (#3644): every way the scanner can fail to run must come + # back as ENV rc=2, never as a row verdict. A must-match fixture is scanned + # each time, so a wrong classification would have to invent "missed hit_use" + # (rc 1) or "no violations" (rc 0) out of an interpreter that never ran. + death() { # death